Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_kafka_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-fxwpv-nsj6n --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "df3a0902e1365a31f68588d9c7197edec59a5322" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-fxwpv" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 5 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=0ae29c97-15f7-4a93-bf48-c7fea539b31b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-fxwpv GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-236df335481f9578f70eb859f68d5ceead3aa27f6c9385fda1ec4c08661c0305 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-fxwpv pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.0' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build3886598031=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7 bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-kafka-test-1872-fxwpv-vf5m7 bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 2622 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 59d8e7b687995bbe (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 20 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Mon May 06 10:53:35 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11024/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11024/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11024/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11024/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [examples/java/pom.xml] [Pipeline] echo diff file not matched: examples/java/pom.xml [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-c710066-880c493) 203906048 bytes in 0.80 secs (256373068 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/.git/ .git HEAD is now at 129b0926c Merge commit '880c4938eb45789c9feb7606b94fbf04e0a60b7a' into HEAD POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/11024/head -> origin/pr/11024/head Warning: you are leaving 1 commit behind, not connected to any of your branches: 129b0926c Merge commit '880c4938eb45789c9feb7606b94fbf04e0a60b7a' into HEAD If you want to keep it by creating a new branch, this may be a good time to do so with: git branch 129b0926c HEAD is now at c710066a5 *(ticdc): split old update kv entry after restarting changefeed (#10919) ๐Ÿšง Checkouting to base SHA:c710066a51e20310bd1ae34af39296cecf567182... HEAD is now at c710066a5 *(ticdc): split old update kv entry after restarting changefeed (#10919) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: c710066a51e20310bd1ae34af39296cecf567182 c710066a5 *(ticdc): split old update kv entry after restarting changefeed (#10919) be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) ๐Ÿšง Pre-merge heads of pull requests to base SHA: c710066a51e20310bd1ae34af39296cecf567182 ... Merge made by the 'ort' strategy. examples/java/pom.xml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) ๐Ÿงพ Pre-merged result: 58799adc6338905a3d6177e5829f44667d1aa09e 58799adc6 Merge commit '880c4938eb45789c9feb7606b94fbf04e0a60b7a' into HEAD c710066a5 *(ticdc): split old update kv entry after restarting changefeed (#10919) 880c4938e Merge branch 'pingcap:master' into fix-java-example-license-issue โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-c710066-880c493 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1827 0 --:--:-- --:--:-- --:--:-- 1863 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2598 0 --:--:-- --:--:-- --:--:-- 2733 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1363 0 --:--:-- --:--:-- --:--:-- 1366 100 41 100 41 0 0 1360 0 --:--:-- --:--:-- --:--:-- 1322 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1194 0 --:--:-- --:--:-- --:--:-- 1205 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1/centos7/tidb-server.tar.gz 2024-05-06 18:53:58 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1/centos7/tidb-server.tar.gz [536591132/536591132] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/f83febabecb98b95b098fd31a3664178f8a6b437/centos7/pd-server.tar.gz 2024-05-06 18:54:10 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/f83febabecb98b95b098fd31a3664178f8a6b437/centos7/pd-server.tar.gz [187381047/187381047] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/88099c95a3c0c13a827c0795c3d45070264665e4/centos7/tikv-server.tar.gz 2024-05-06 18:54:49 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/88099c95a3c0c13a827c0795c3d45070264665e4/centos7/tikv-server.tar.gz [919298452/919298452] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz 2024-05-06 18:55:16 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e50de84e6d6ecdcc108990217b70b6bb3f50271/centos7/tiflash.tar.gz [456051121/456051121] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-06 18:55:21 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-06 18:55:24 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-06 18:55:25 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-06 18:55:25 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-06 18:55:30 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-06 18:55:46 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 18:55 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 18:55 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 18:55 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 18:55 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 37M May 6 16:50 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M May 6 16:49 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M May 6 16:50 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:49 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M May 6 16:49 pd-server -rwxr-xr-x 1 jenkins jenkins 26M May 6 16:50 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M May 6 16:50 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:50 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:50 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 6 18:31 tidb-server -rwxr-xr-x 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x 1 jenkins jenkins 418M May 6 15:33 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M May 6 16:50 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 18:55 . drwxr-sr-x 3 jenkins jenkins 4.0K May 6 18:55 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 18:55 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 18:55 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 12:44 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 12:44 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 12:44 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 12:05 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 272M May 6 12:45 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x 1 jenkins jenkins 37M May 6 16:50 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M May 6 16:49 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M May 6 16:50 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:49 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M May 6 16:49 pd-server -rwxr-xr-x 1 jenkins jenkins 26M May 6 16:50 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M May 6 16:50 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:50 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x 1 jenkins jenkins 32M May 6 16:50 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 6 18:31 tidb-server -rwxr-xr-x 1 jenkins jenkins 380M May 6 12:44 tiflash -rwxr-xr-x 1 jenkins jenkins 418M May 6 15:33 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M May 6 16:50 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x 2 jenkins jenkins 4.0K May 6 18:55 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 18:55 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g58799adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:55:50" -X "github.com/pingcap/tiflow/pkg/version.GitHash=58799adc6338905a3d6177e5829f44667d1aa09e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g58799adc6"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading golang.org/x/net v0.24.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading gorm.io/gorm v1.24.5 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/tools v0.20.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g58799adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:57:56" -X "github.com/pingcap/tiflow/pkg/version.GitHash=58799adc6338905a3d6177e5829f44667d1aa09e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g58799adc6"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g58799adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:58:03" -X "github.com/pingcap/tiflow/pkg/version.GitHash=58799adc6338905a3d6177e5829f44667d1aa09e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g58799adc6"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g58799adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:58:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=58799adc6338905a3d6177e5829f44667d1aa09e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g58799adc6"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g58799adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:58:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=58799adc6338905a3d6177e5829f44667d1aa09e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g58799adc6"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g58799adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:58:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=58799adc6338905a3d6177e5829f44667d1aa09e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g58799adc6"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g58799adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:58:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=58799adc6338905a3d6177e5829f44667d1aa09e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g58799adc6"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g58799adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:58:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=58799adc6338905a3d6177e5829f44667d1aa09e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g58799adc6"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-25-g58799adc6" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-06 10:58:10" -X "github.com/pingcap/tiflow/pkg/version.GitHash=58799adc6338905a3d6177e5829f44667d1aa09e" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-25-g58799adc6"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x 2 jenkins jenkins 4.0K May 6 19:02 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 18:55 .. -rwxr-xr-x 1 jenkins jenkins 220M May 6 19:02 cdc -rwxr-xr-x 1 jenkins jenkins 359M May 6 19:02 cdc.test -rwxr-xr-x 1 jenkins jenkins 183M May 6 18:58 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 183M May 6 18:58 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 18:58 cdc_storage_consumer -rwxr-xr-x 1 jenkins jenkins 12M May 6 18:58 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-25-g58799adc6 Git Commit Hash: 58799adc6338905a3d6177e5829f44667d1aa09e Git Branch: HEAD UTC Build Time: 2024-05-06 10:58:10 Go Version: go version go1.21.0 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-test/rev-c710066-880c493) 1191699968 bytes in 16.99 secs (70148500 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x 6 jenkins jenkins 4.0K May 6 19:03 . drwxr-sr-x 19 jenkins jenkins 4.0K May 6 18:55 .. drwxr-sr-x 2 jenkins jenkins 4.0K May 6 19:03 bin -rwxr-xr-x 1 jenkins jenkins 220M May 6 19:02 cdc -rwxr-xr-x 1 jenkins jenkins 359M May 6 19:02 cdc.test -rwxr-xr-x 1 jenkins jenkins 183M May 6 18:58 cdc_kafka_consumer -rwxr-xr-x 1 jenkins jenkins 183M May 6 18:58 cdc_pulsar_consumer -rwxr-xr-x 1 jenkins jenkins 182M May 6 18:58 cdc_storage_consumer drwxr-sr-x 4 jenkins jenkins 4.0K May 6 19:03 etc -rwxr-xr-x 1 jenkins jenkins 17M May 6 19:03 etcdctl -rwxr-xr-x 1 jenkins jenkins 44M May 6 19:03 go-ycsb -rwxr-xr-x 1 jenkins jenkins 3.8M May 6 19:03 jq drwxr-sr-x 3 jenkins jenkins 4.0K May 6 19:03 lib lrwxrwxrwx 1 jenkins jenkins 13 May 6 19:03 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x 1 jenkins jenkins 1016K May 6 19:03 libc++.so.1.0 lrwxrwxrwx 1 jenkins jenkins 16 May 6 19:03 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x 1 jenkins jenkins 358K May 6 19:03 libc++abi.so.1.0 lrwxrwxrwx 1 jenkins jenkins 13 May 6 19:03 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx 1 jenkins jenkins 15 May 6 19:03 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 2.6M May 6 19:03 libgmssl.so.3.0 -rwxr-xr-x 1 jenkins jenkins 272M May 6 19:03 libtiflash_proxy.so -rwxr-xr-x 1 jenkins jenkins 50M May 6 19:03 minio -rwxr-xr-x 1 jenkins jenkins 12M May 6 18:58 oauth2-server -rwxr-xr-x 1 jenkins jenkins 37M May 6 19:03 pd-api-bench -rwxr-xr-x 1 jenkins jenkins 44M May 6 19:03 pd-ctl -rwxr-xr-x 1 jenkins jenkins 36M May 6 19:03 pd-heartbeat-bench -rwxr-xr-x 1 jenkins jenkins 32M May 6 19:03 pd-recover -rwxr-xr-x 1 jenkins jenkins 106M May 6 19:03 pd-server -rwxr-xr-x 1 jenkins jenkins 26M May 6 19:03 pd-tso-bench -rwxr-xr-x 1 jenkins jenkins 3.0M May 6 19:03 pd-ut -rwxr-xr-x 1 jenkins jenkins 32M May 6 19:03 regions-dump drwxr-sr-x 4 jenkins jenkins 4.0K May 6 19:03 share -rwxr-xr-x 1 jenkins jenkins 32M May 6 19:03 stores-dump -rwxr-xr-x 1 jenkins jenkins 192M May 6 19:03 sync_diff_inspector -rwxr-xr-x 1 jenkins jenkins 208M May 6 19:03 tidb-server -rwxr-xr-x 1 jenkins jenkins 380M May 6 19:03 tiflash -rwxr-xr-x 1 jenkins jenkins 418M May 6 19:03 tikv-server -rwxr-xr-x 1 jenkins jenkins 2.0M May 6 19:03 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 101.72 secs (36644456 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-39c29-z2sqd --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "01a95b84d384102a165f04262c0d45c985f189b2" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-39c29" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-3z6n6-982qz --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7515c7377d3a7260670bdcf7edf1421a88edf3ad" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-3z6n6" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-hfkv1-sj5pq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "387e3c79c461248f91db255020092f9178a37d7d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-hfkv1" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-gs6jt-v5kb2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7acf364a3739ae4b4bb60d5800e8777db35c7390" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-gs6jt" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-c82s3-5d70g --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "871c46fae96c336cac9493b3d8e32d790d66ed6c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-c82s3" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@49a999ca; decorates RemoteLauncher[hudson.remoting.Channel@528db31c:JNLP4-connect connection from 10.233.67.241/10.233.67.241:37258] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2 Cloning repository https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6cfcbf59; decorates RemoteLauncher[hudson.remoting.Channel@2c64b222:JNLP4-connect connection from 10.233.73.235/10.233.73.235:36714] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@28331f85; decorates RemoteLauncher[hudson.remoting.Channel@2ab1e5d6:JNLP4-connect connection from 10.233.70.158/10.233.70.158:46838] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@65b46d8c; decorates RemoteLauncher[hudson.remoting.Channel@65b2866b:JNLP4-connect connection from 10.233.105.220/10.233.105.220:35078] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7dc937cf; decorates RemoteLauncher[hudson.remoting.Channel@c279cea:JNLP4-connect connection from 10.233.108.137/10.233.108.137:41288] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9 > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-lhjzd-h2fmg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5327109998bfcd6c7730503ef69a36dda981e487" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-lhjzd" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z [Pipeline] node > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4ce5514d; decorates RemoteLauncher[hudson.remoting.Channel@36ae29df:JNLP4-connect connection from 10.233.123.80/10.233.123.80:51144] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8k3vn-qlz3k --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "37b2c5764935eb0866e0f124adfe338cfa8bc68d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8k3vn" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout Commit message: "fix(br): use failpoint tidb-server instead (#2951)" The recommended git tool is: git [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] podTemplate [Pipeline] { [Pipeline] cache > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6fea5f67; decorates RemoteLauncher[hudson.remoting.Channel@1e67befc:JNLP4-connect connection from 10.233.107.180/10.233.107.180:59454] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8m3lr-pm3jt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5739eddf1146c28285e5ca5dc608b05e31d2d1de" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8m3lr" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-k2r9w-6lz7h --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "12572bec8a0839e47d4f4d7f9be70c9207380921" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-k2r9w" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8bdr9-147sx --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d05b8d08864ee55fce4a78b52fcd6c537d6358f0" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8bdr9" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-397v7-t4lp8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "aa4108c913b2e4611f87d4598f653296b754fda2" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-397v7" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-qfjm4-szn0b --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "306ebb9115eff962911d1ada10f84265b51fef2f" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-qfjm4" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-ds960-q48r5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f94f1f71d2d8d4d1b52a74de7f3276f1cb62dfc5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-ds960" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 25.37 secs (146909636 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] node [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] sh [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-0kfbr [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-qql82 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-lj2k4 The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-t6bnd [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // timeout [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // container [Pipeline] sh No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@67ff8ee2; decorates RemoteLauncher[hudson.remoting.Channel@3555bda2:JNLP4-connect connection from 10.233.84.78/10.233.84.78:41594] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4604a1c6; decorates RemoteLauncher[hudson.remoting.Channel@3bd30e9c:JNLP4-connect connection from 10.233.68.10/10.233.68.10:36814] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3bdec774; decorates RemoteLauncher[hudson.remoting.Channel@3f726215:JNLP4-connect connection from 10.233.97.139/10.233.97.139:52940] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@798d27c3; decorates RemoteLauncher[hudson.remoting.Channel@342b901:JNLP4-connect connection from 10.233.126.77/10.233.126.77:41702] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2cf2bd5d; decorates RemoteLauncher[hudson.remoting.Channel@3296df4f:JNLP4-connect connection from 10.233.66.3/10.233.66.3:45824] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3030a310; decorates RemoteLauncher[hudson.remoting.Channel@4b185930:JNLP4-connect connection from 10.233.69.156/10.233.69.156:33150] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=08a56bb6-1a70-49df-b8df-30acc1f6cfe2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-39c29 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-39c29 pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:05:42 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCE18B9686F52F < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 11:05:47 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Mon May 6 19:05:47 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:05:51 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:05:54 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-0kfbrโ€™ is offline Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514โ€™ is offline Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-qql82โ€™ is offline Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-lj2k4โ€™ is offline Still waiting to schedule task โ€˜pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-t6bndโ€™ is offline find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:05:57 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCE18F36528306 < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 11:06:03 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Mon May 6 19:06:03 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:06:06 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365568080003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5, pid:2113, start at 2024-05-06 19:06:24.644054321 +0800 CST m=+5.305604503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:08:24.651 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:06:24.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:56:24.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365568080003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5, pid:2113, start at 2024-05-06 19:06:24.644054321 +0800 CST m=+5.305604503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:08:24.651 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:06:24.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:56:24.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365567280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5, pid:2191, start at 2024-05-06 19:06:24.611510444 +0800 CST m=+5.217594756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:08:24.620 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:06:24.586 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:56:24.586 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 19:06:29 CST 2024] <<<<<< START cdc server in kafka_big_messages_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages_v2.36193621.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 46.99 secs (79328985 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:06:32 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91abf279-64e3-4637-901f-926dff50514b {"id":"91abf279-64e3-4637-901f-926dff50514b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d952f67cd 91abf279-64e3-4637-901f-926dff50514b /tidb/cdc/default/default/upstream/7365841324676480278 {"id":7365841324676480278,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91abf279-64e3-4637-901f-926dff50514b {"id":"91abf279-64e3-4637-901f-926dff50514b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d952f67cd 91abf279-64e3-4637-901f-926dff50514b /tidb/cdc/default/default/upstream/7365841324676480278 {"id":7365841324676480278,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/91abf279-64e3-4637-901f-926dff50514b {"id":"91abf279-64e3-4637-901f-926dff50514b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d952f67cd 91abf279-64e3-4637-901f-926dff50514b /tidb/cdc/default/default/upstream/7365841324676480278 {"id":7365841324676480278,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: b1633f99-8c24-477f-a3c5-449fc034d920 Info: {"upstream_id":7365841324676480278,"namespace":"default","id":"b1633f99-8c24-477f-a3c5-449fc034d920","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-05-06T19:06:33.042391722+08:00","start_ts":449575279589326849,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575279589326849,"checkpoint_ts":449575279589326849,"checkpoint_time":"2024-05-06 19:06:29.742"} [Mon May 6 19:06:33 CST 2024] <<<<<< START kafka consumer in kafka_big_messages_v2 case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 table kafka_big_messages.test exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:06:46 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 14.66 secs (254313382 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 20.68 secs (180266113 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365853d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5, pid:6034, start at 2024-05-06 19:07:12.538464964 +0800 CST m=+5.449940995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:09:12.547 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:07:12.551 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:57:12.551 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365853d40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5, pid:6034, start at 2024-05-06 19:07:12.538464964 +0800 CST m=+5.449940995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:09:12.547 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:07:12.551 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:57:12.551 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365856300009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5, pid:6116, start at 2024-05-06 19:07:12.665380257 +0800 CST m=+5.518589267 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:09:12.673 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:07:12.652 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:57:12.652 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 19:07:17 CST 2024] <<<<<< START cdc server in multi_tables_ddl_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl_v2.75027504.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:07:20 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ead1ce6b-def5-4229-811f-bd75daae9e76 {"id":"ead1ce6b-def5-4229-811f-bd75daae9e76","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d95e9c5d2 ead1ce6b-def5-4229-811f-bd75daae9e76 /tidb/cdc/default/default/upstream/7365841518867936397 {"id":7365841518867936397,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ead1ce6b-def5-4229-811f-bd75daae9e76 {"id":"ead1ce6b-def5-4229-811f-bd75daae9e76","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d95e9c5d2 ead1ce6b-def5-4229-811f-bd75daae9e76 /tidb/cdc/default/default/upstream/7365841518867936397 {"id":7365841518867936397,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ead1ce6b-def5-4229-811f-bd75daae9e76 {"id":"ead1ce6b-def5-4229-811f-bd75daae9e76","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d95e9c5d2 ead1ce6b-def5-4229-811f-bd75daae9e76 /tidb/cdc/default/default/upstream/7365841518867936397 {"id":7365841518867936397,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7365841518867936397,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-4068?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:07:20.871182145+08:00","start_ts":449575292109062149,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575292109062149,"checkpoint_ts":449575292109062149,"checkpoint_time":"2024-05-06 19:07:17.501"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7365841518867936397,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-32584?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:07:21.11138141+08:00","start_ts":449575292109062149,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575292109062149,"checkpoint_ts":449575292109062149,"checkpoint_time":"2024-05-06 19:07:17.501"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7365841518867936397,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-21840?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:07:21.315674215+08:00","start_ts":449575292109062149,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575292109062149,"checkpoint_ts":449575292109062149,"checkpoint_time":"2024-05-06 19:07:17.501"} [Mon May 6 19:07:21 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Mon May 6 19:07:21 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Mon May 6 19:07:21 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 18.63 secs (200115972 bytes/sec) [Pipeline] { [Pipeline] cache table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7365841518867936397, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449575293930962978, "checkpoint_time": "2024-05-06 19:07:24.451", "error": null }' + echo '{ "upstream_id": 7365841518867936397, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449575293930962978, "checkpoint_time": "2024-05-06 19:07:24.451", "error": null }' { "upstream_id": 7365841518867936397, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449575293930962978, "checkpoint_time": "2024-05-06 19:07:24.451", "error": null } ++ echo '{' '"upstream_id":' 7365841518867936397, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449575293930962978, '"checkpoint_time":' '"2024-05-06' '19:07:24.451",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365841518867936397, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449575293930962978, '"checkpoint_time":' '"2024-05-06' '19:07:24.451",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7365841518867936397, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449575297011154945, "checkpoint_time": "2024-05-06 19:07:36.201", "error": null }' + echo '{ "upstream_id": 7365841518867936397, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449575297011154945, "checkpoint_time": "2024-05-06 19:07:36.201", "error": null }' { "upstream_id": 7365841518867936397, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449575297011154945, "checkpoint_time": "2024-05-06 19:07:36.201", "error": null } ++ echo '{' '"upstream_id":' 7365841518867936397, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449575297011154945, '"checkpoint_time":' '"2024-05-06' '19:07:36.201",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365841518867936397, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449575297011154945, '"checkpoint_time":' '"2024-05-06' '19:07:36.201",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7365841518867936397, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449575293432889381, "checkpoint_time": "2024-05-06 19:07:22.551", "error": { "time": "2024-05-06T19:07:25.245860213+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7365841518867936397, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449575293432889381, "checkpoint_time": "2024-05-06 19:07:22.551", "error": { "time": "2024-05-06T19:07:25.245860213+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7365841518867936397, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449575293432889381, "checkpoint_time": "2024-05-06 19:07:22.551", "error": { "time": "2024-05-06T19:07:25.245860213+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365841518867936397, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575293432889381, '"checkpoint_time":' '"2024-05-06' '19:07:22.551",' '"error":' '{' '"time":' '"2024-05-06T19:07:25.245860213+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365841518867936397, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575293432889381, '"checkpoint_time":' '"2024-05-06' '19:07:22.551",' '"error":' '{' '"time":' '"2024-05-06T19:07:25.245860213+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:07:39 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 8.94 secs (417060306 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_topics_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 17.59 secs (211939427 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365b8fa40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5, pid:9185, start at 2024-05-06 19:08:05.516008471 +0800 CST m=+5.435071249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:05.526 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:05.531 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:05.531 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365b8fa40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5, pid:9185, start at 2024-05-06 19:08:05.516008471 +0800 CST m=+5.435071249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:05.526 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:05.531 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:05.531 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365b92d40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-39c29-9jsb5, pid:9270, start at 2024-05-06 19:08:05.724174406 +0800 CST m=+5.579708720 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:05.732 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:05.735 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:05.735 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.10612.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // timeout [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] sh + set +x + tso='449575306076094465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575306076094465 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:08:12 CST 2024] <<<<<< START cdc server in multi_topics_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.1064510647.out server --log-file /tmp/tidb_cdc_test/multi_topics_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics_v2/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G01 [Pipeline] sh Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=69d9e875-feed-4c17-ac17-a421fc970347 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-3z6n6 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-3z6n6 pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:08:13 CST 2024] <<<<<< run test case http_api success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=a107cc20-c560-457d-8615-e4c9a30c7b15 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-hfkv1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j pingcap_tiflow_pull_cdc_integration_kafka_test_1872-hfkv1 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G00 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=3f39067e-f508-40b3-975f-cb2cf8f81518 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-c82s3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-c82s3 pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=86119458-a58f-406b-ae75-cd63ebf14f1c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-gs6jt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-gs6jt pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:08:13 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=7b9eca3b-ff57-42ce-85e9-16436bdc3962 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-lhjzd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0 pingcap_tiflow_pull_cdc_integration_kafka_test_1872-lhjzd GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=63503712-5adb-456c-85c7-b590bd428654 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8k3vn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8k3vn pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:15 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53895d14-7bda-4862-a530-590d0b9aba58 {"id":"53895d14-7bda-4862-a530-590d0b9aba58","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993692} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96badfd5 53895d14-7bda-4862-a530-590d0b9aba58 /tidb/cdc/default/default/upstream/7365841753305346335 {"id":7365841753305346335,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53895d14-7bda-4862-a530-590d0b9aba58 {"id":"53895d14-7bda-4862-a530-590d0b9aba58","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993692} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96badfd5 53895d14-7bda-4862-a530-590d0b9aba58 /tidb/cdc/default/default/upstream/7365841753305346335 {"id":7365841753305346335,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53895d14-7bda-4862-a530-590d0b9aba58 {"id":"53895d14-7bda-4862-a530-590d0b9aba58","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993692} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d96badfd5 53895d14-7bda-4862-a530-590d0b9aba58 /tidb/cdc/default/default/upstream/7365841753305346335 {"id":7365841753305346335,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.10702.out cli changefeed create --start-ts=449575306076094465 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/conf/changefeed.toml Create changefeed successfully! ID: 1a04aea5-2b49-44d1-994b-d24acb16ceb3 Info: {"upstream_id":7365841753305346335,"namespace":"default","id":"1a04aea5-2b49-44d1-994b-d24acb16ceb3","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-05-06T19:08:15.878262091+08:00","start_ts":449575306076094465,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575306076094465,"checkpoint_ts":449575306076094465,"checkpoint_time":"2024-05-06 19:08:10.781"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:08:18 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:08:21 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: kafka... <<================= Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: kafka... <<================= kafka downstream isn't support syncpoint record [Mon May 6 19:08:26 CST 2024] <<<<<< run test case syncpoint success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365cd4bc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:1355, start at 2024-05-06 19:08:26.290237964 +0800 CST m=+5.162385322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:26.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:26.287 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:26.287 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365cd4bc0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:1355, start at 2024-05-06 19:08:26.290237964 +0800 CST m=+5.162385322 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:26.297 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:26.287 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:26.287 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365cd5740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:1436, start at 2024-05-06 19:08:26.355421733 +0800 CST m=+5.170864231 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:26.361 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:26.333 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:26.333 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:08:27 CST 2024] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 11.32 secs (329387457 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2837.out cli tso query --pd=http://127.0.0.1:2379 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:08:29 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365d05d00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:1412, start at 2024-05-06 19:08:29.466453723 +0800 CST m=+5.208489269 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:29.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:29.478 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:29.478 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365d05d00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:1412, start at 2024-05-06 19:08:29.466453723 +0800 CST m=+5.208489269 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:29.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:29.478 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:29.478 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365d075c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:1491, start at 2024-05-06 19:08:29.556546617 +0800 CST m=+5.239354847 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:29.565 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:29.527 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:29.527 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449575311058403329 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575311058403329 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:08:31 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.28752877.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365cfd400012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0, pid:1349, start at 2024-05-06 19:08:28.899576983 +0800 CST m=+5.172594423 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:28.905 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:28.880 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:28.880 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365cfd400012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0, pid:1349, start at 2024-05-06 19:08:28.899576983 +0800 CST m=+5.172594423 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:28.905 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:28.880 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:28.880 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365cfec40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0, pid:1435, start at 2024-05-06 19:08:29.014725975 +0800 CST m=+5.236753211 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:29.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:28.977 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:28.977 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365d084c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:1407, start at 2024-05-06 19:08:29.625318287 +0800 CST m=+5.180006698 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:29.633 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:29.637 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:29.637 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365d084c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:1407, start at 2024-05-06 19:08:29.625318287 +0800 CST m=+5.180006698 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:29.633 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:29.637 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:29.637 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365d09340014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:1489, start at 2024-05-06 19:08:29.687333239 +0800 CST m=+5.186005234 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:29.696 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:29.695 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:29.695 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:08:32 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2820.out cli tso query --pd=http://127.0.0.1:2379 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x + tso='449575311881797633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575311881797633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:08:34 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.28302832.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:34 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b5c99d6-0676-4753-8e39-e5dde167667f {"id":"0b5c99d6-0676-4753-8e39-e5dde167667f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993711} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d970f16cb 0b5c99d6-0676-4753-8e39-e5dde167667f /tidb/cdc/default/default/upstream/7365841849379756043 {"id":7365841849379756043,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b5c99d6-0676-4753-8e39-e5dde167667f {"id":"0b5c99d6-0676-4753-8e39-e5dde167667f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993711} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d970f16cb 0b5c99d6-0676-4753-8e39-e5dde167667f /tidb/cdc/default/default/upstream/7365841849379756043 {"id":7365841849379756043,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b5c99d6-0676-4753-8e39-e5dde167667f {"id":"0b5c99d6-0676-4753-8e39-e5dde167667f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993711} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d970f16cb 0b5c99d6-0676-4753-8e39-e5dde167667f /tidb/cdc/default/default/upstream/7365841849379756043 {"id":7365841849379756043,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2927.out cli changefeed create --start-ts=449575311058403329 '--sink-uri=kafka://127.0.0.1:9092/ticdc-row-format-test-7292?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' ***************** properties ***************** "dotransactions"="false" "threadcount"="2" "workload"="core" "requestdistribution"="uniform" "recordcount"="10" "operationcount"="0" "readproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "readallfields"="true" "mysql.user"="root" "insertproportion"="0" "updateproportion"="0" "mysql.db"="sink_retry" ********************************************** Run finished, takes 11.728162ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1432.0, Avg(us): 2265, Min(us): 1409, Max(us): 4615, 95th(us): 5000, 99th(us): 5000 [Mon May 6 19:08:34 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.28782880.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: e014ecfa-f9ce-4a3a-838d-c6cbf58fbdb1 Info: {"upstream_id":7365841849379756043,"namespace":"default","id":"e014ecfa-f9ce-4a3a-838d-c6cbf58fbdb1","sink_uri":"kafka://127.0.0.1:9092/ticdc-row-format-test-7292?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:08:34.832401935+08:00","start_ts":449575311058403329,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575311058403329,"checkpoint_ts":449575311058403329,"checkpoint_time":"2024-05-06 19:08:29.787"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2815.out cli tso query --pd=http://127.0.0.1:2379 + set +x [Mon May 6 19:08:36 CST 2024] <<<<<< START kafka consumer in row_format case >>>>>> + set +x + tso='449575312460873729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575312460873729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:08:36 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.28522854.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:37 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/833d2a7c-abc2-4c53-9b80-49193a2e113c {"id":"833d2a7c-abc2-4c53-9b80-49193a2e113c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993714} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d971359cd 833d2a7c-abc2-4c53-9b80-49193a2e113c /tidb/cdc/default/default/upstream/7365841856066985004 {"id":7365841856066985004,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/833d2a7c-abc2-4c53-9b80-49193a2e113c {"id":"833d2a7c-abc2-4c53-9b80-49193a2e113c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993714} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d971359cd 833d2a7c-abc2-4c53-9b80-49193a2e113c /tidb/cdc/default/default/upstream/7365841856066985004 {"id":7365841856066985004,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/833d2a7c-abc2-4c53-9b80-49193a2e113c {"id":"833d2a7c-abc2-4c53-9b80-49193a2e113c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993714} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d971359cd 833d2a7c-abc2-4c53-9b80-49193a2e113c /tidb/cdc/default/default/upstream/7365841856066985004 {"id":7365841856066985004,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 2bf3316d-46db-420a-80e2-c6e00c3f58e1 Info: {"upstream_id":7365841856066985004,"namespace":"default","id":"2bf3316d-46db-420a-80e2-c6e00c3f58e1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-06T19:08:37.438600159+08:00","start_ts":449575312197156865,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575312197156865,"checkpoint_ts":449575312197156865,"checkpoint_time":"2024-05-06 19:08:34.131"} [Mon May 6 19:08:37 CST 2024] <<<<<< START kafka consumer in charset_gbk case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:37 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9527c6fe-72b5-4558-95ed-0ce04984c20e {"id":"9527c6fe-72b5-4558-95ed-0ce04984c20e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993715} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d971b29e8 9527c6fe-72b5-4558-95ed-0ce04984c20e /tidb/cdc/default/default/upstream/7365841854431331846 {"id":7365841854431331846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9527c6fe-72b5-4558-95ed-0ce04984c20e {"id":"9527c6fe-72b5-4558-95ed-0ce04984c20e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993715} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d971b29e8 9527c6fe-72b5-4558-95ed-0ce04984c20e /tidb/cdc/default/default/upstream/7365841854431331846 {"id":7365841854431331846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9527c6fe-72b5-4558-95ed-0ce04984c20e {"id":"9527c6fe-72b5-4558-95ed-0ce04984c20e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993715} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d971b29e8 9527c6fe-72b5-4558-95ed-0ce04984c20e /tidb/cdc/default/default/upstream/7365841854431331846 {"id":7365841854431331846,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2937.out cli changefeed create --start-ts=449575311881797633 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-25878?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:08:37 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> Create changefeed successfully! ID: 40aab2a3-d880-4bf2-8a5b-554081e03300 Info: {"upstream_id":7365841854431331846,"namespace":"default","id":"40aab2a3-d880-4bf2-8a5b-554081e03300","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-25878?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:08:38.216618529+08:00","start_ts":449575311881797633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575311881797633,"checkpoint_ts":449575311881797633,"checkpoint_time":"2024-05-06 19:08:32.928"} PASS Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... 2024-05-06 19:08:38.181570 W | etcdserver: could not get cluster response from http://127.0.0.1:2380: Get "http://127.0.0.1:2380/members": dial tcp 127.0.0.1:2380: connect: connection refused 2024-05-06 19:08:38.209542 W | etcdserver: could not get cluster response from http://127.0.0.1:2380: Get "http://127.0.0.1:2380/members": dial tcp 127.0.0.1:2380: connect: connection refused coverage: 2.4% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 7.51 secs (496499226 bytes/sec) [Pipeline] { [Pipeline] cache + set +x [Mon May 6 19:08:39 CST 2024] <<<<<< START kafka consumer in sink_retry case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:08:39 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c6364d0-a0c2-430a-9369-89c63e46643f {"id":"6c6364d0-a0c2-430a-9369-89c63e46643f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d971271d4 6c6364d0-a0c2-430a-9369-89c63e46643f /tidb/cdc/default/default/upstream/7365841859001069568 {"id":7365841859001069568,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c6364d0-a0c2-430a-9369-89c63e46643f {"id":"6c6364d0-a0c2-430a-9369-89c63e46643f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d971271d4 6c6364d0-a0c2-430a-9369-89c63e46643f /tidb/cdc/default/default/upstream/7365841859001069568 {"id":7365841859001069568,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c6364d0-a0c2-430a-9369-89c63e46643f {"id":"6c6364d0-a0c2-430a-9369-89c63e46643f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d971271d4 6c6364d0-a0c2-430a-9369-89c63e46643f /tidb/cdc/default/default/upstream/7365841859001069568 {"id":7365841859001069568,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2917.out cli changefeed create --start-ts=449575312460873729 '--sink-uri=kafka://127.0.0.1:9092/ticdc-foreign-key-test-30667?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 30bb6d65-5fbc-4e22-b2d6-979c42629d3f Info: {"upstream_id":7365841859001069568,"namespace":"default","id":"30bb6d65-5fbc-4e22-b2d6-979c42629d3f","sink_uri":"kafka://127.0.0.1:9092/ticdc-foreign-key-test-30667?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:08:40.235059208+08:00","start_ts":449575312460873729,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575312460873729,"checkpoint_ts":449575312460873729,"checkpoint_time":"2024-05-06 19:08:35.137"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x [Mon May 6 19:08:41 CST 2024] <<<<<< START kafka consumer in foreign_key case >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 1-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 2-th check, retry later table row_format.finish_mark not exists for 4-th check, retry later table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... table test.finish_mark not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365e64600001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:1678, start at 2024-05-06 19:08:51.865354746 +0800 CST m=+8.322217658 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:51.879 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:51.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:51.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365e64600001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:1678, start at 2024-05-06 19:08:51.865354746 +0800 CST m=+8.322217658 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:51.879 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:51.864 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:51.864 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365e6344000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:1705, start at 2024-05-06 19:08:51.826388362 +0800 CST m=+8.140409113 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:10:51.854 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:08:51.843 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:58:51.843 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table row_format.finish_mark not exists for 5-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table foreign_key.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 6-th check, retry later table foreign_key.finish_mark not exists for 4-th check, retry later The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 7-th check, retry later wait process cdc.test exit for 2-th time... table foreign_key.finish_mark not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:08:58 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> table row_format.finish_mark not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 6-th check, retry later Starting TLS TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365eee0c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:1917, start at 2024-05-06 19:09:00.697771737 +0800 CST m=+5.139068179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:00.704 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:00.675 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:00.675 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365eee0c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:1917, start at 2024-05-06 19:09:00.697771737 +0800 CST m=+5.139068179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:00.704 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:00.675 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:00.675 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365eef9c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:2006, start at 2024-05-06 19:09:00.783427371 +0800 CST m=+5.167872402 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:00.789 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:00.775 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:00.775 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table row_format.finish_mark not exists for 9-th check, retry later table foreign_key.finish_mark not exists for 7-th check, retry later Starting TLS TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 10-th check, retry later table foreign_key.finish_mark not exists for 8-th check, retry later [Mon May 6 19:09:03 CST 2024] <<<<<< START cdc server in kafka_big_messages case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages.33973399.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... table foreign_key.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:07 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6a56b29-10ce-422f-beeb-7781a438bfc7 {"id":"c6a56b29-10ce-422f-beeb-7781a438bfc7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993744} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d978f8ccb c6a56b29-10ce-422f-beeb-7781a438bfc7 /tidb/cdc/default/default/upstream/7365841987809344380 {"id":7365841987809344380,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6a56b29-10ce-422f-beeb-7781a438bfc7 {"id":"c6a56b29-10ce-422f-beeb-7781a438bfc7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993744} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d978f8ccb c6a56b29-10ce-422f-beeb-7781a438bfc7 /tidb/cdc/default/default/upstream/7365841987809344380 {"id":7365841987809344380,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' wait process cdc.test exit for 2-th time... + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c6a56b29-10ce-422f-beeb-7781a438bfc7 {"id":"c6a56b29-10ce-422f-beeb-7781a438bfc7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993744} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d978f8ccb c6a56b29-10ce-422f-beeb-7781a438bfc7 /tidb/cdc/default/default/upstream/7365841987809344380 {"id":7365841987809344380,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: a11902bc-d63e-4ea2-8cce-73df30d216b4 Info: {"upstream_id":7365841987809344380,"namespace":"default","id":"a11902bc-d63e-4ea2-8cce-73df30d216b4","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-05-06T19:09:07.270152324+08:00","start_ts":449575320007737345,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575320007737345,"checkpoint_ts":449575320007737345,"checkpoint_time":"2024-05-06 19:09:03.926"} [Mon May 6 19:09:07 CST 2024] <<<<<< START kafka consumer in kafka_big_messages case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:09:07 CST 2024] <<<<<< run test case row_format success! >>>>>> wait process cdc.test exit for 2-th time... go: downloading go.uber.org/atomic v1.11.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:09:07 CST 2024] <<<<<< run test case foreign_key success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 28.05 secs (132905567 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d365f8708000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:3278, start at 2024-05-06 19:09:10.502987049 +0800 CST m=+7.526851958 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:10.544 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:10.564 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:10.564 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3331.out cli tso query --pd=https://127.0.0.1:2579 table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "insertproportion"="0" "readallfields"="true" "readproportion"="0" "mysql.port"="4000" "workload"="core" "operationcount"="0" "mysql.db"="sink_retry" "mysql.user"="root" "dotransactions"="false" "recordcount"="10" "updateproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "scanproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 4.764863ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3204.1, Avg(us): 894, Min(us): 559, Max(us): 1775, 95th(us): 2000, 99th(us): 2000 start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x + tso='449575322089160705 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575322089160705 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table kafka_big_messages.test not exists for 1-th check, retry later [Mon May 6 19:09:14 CST 2024] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates ']' + GO_FAILPOINTS= + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.336133633365.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table kafka_big_messages.test exists check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:17 GMT < Content-Length: 1233 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/690bfe27-5b89-4723-8b6a-c29ec5d62fa4 {"id":"690bfe27-5b89-4723-8b6a-c29ec5d62fa4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993754} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4d97ab618f 690bfe27-5b89-4723-8b6a-c29ec5d62fa4 /tidb/cdc/default/default/upstream/7365842025001456188 {"id":7365842025001456188,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/690bfe27-5b89-4723-8b6a-c29ec5d62fa4 {"id":"690bfe27-5b89-4723-8b6a-c29ec5d62fa4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993754} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4d97ab618f 690bfe27-5b89-4723-8b6a-c29ec5d62fa4 /tidb/cdc/default/default/upstream/7365842025001456188 {"id":7365842025001456188,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/690bfe27-5b89-4723-8b6a-c29ec5d62fa4 {"id":"690bfe27-5b89-4723-8b6a-c29ec5d62fa4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993754} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f4d97ab618f 690bfe27-5b89-4723-8b6a-c29ec5d62fa4 /tidb/cdc/default/default/upstream/7365842025001456188 {"id":7365842025001456188,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:09:19 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3400.out cli changefeed create --start-ts=449575322089160705 '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-27012?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365842025001456188,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-27012?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:09:20.645334611+08:00","start_ts":449575322089160705,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575322089160705,"checkpoint_ts":449575322089160705,"checkpoint_time":"2024-05-06 19:09:11.866"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x [Mon May 6 19:09:22 CST 2024] <<<<<< START kafka consumer in cli_tls_with_auth case >>>>>> table test.simple not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366064000012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0, pid:4055, start at 2024-05-06 19:09:24.629975368 +0800 CST m=+5.328238905 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:24.636 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:24.608 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:24.608 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.simple exists table test.`simple-dash` exists Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575322665877511, "checkpoint_time": "2024-05-06 19:09:14.066", "error": null }' + echo '{ "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575322665877511, "checkpoint_time": "2024-05-06 19:09:14.066", "error": null }' { "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575322665877511, "checkpoint_time": "2024-05-06 19:09:14.066", "error": null } ++ echo '{' '"upstream_id":' 7365842025001456188, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575322665877511, '"checkpoint_time":' '"2024-05-06' '19:09:14.066",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842025001456188, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575322665877511, '"checkpoint_time":' '"2024-05-06' '19:09:14.066",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366064000012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0, pid:4055, start at 2024-05-06 19:09:24.629975368 +0800 CST m=+5.328238905 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:24.636 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:24.608 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:24.608 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366064cc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0, pid:4145, start at 2024-05-06 19:09:24.678789849 +0800 CST m=+5.324173454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:24.686 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:24.659 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:24.659 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc002de89d0} {Type:update Path:[Config SyncPointRetention] From: To:0xc002de89d8} {Type:update Path:[Config Consistent] From: To:0xc0010c3dc0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3548.out cli changefeed --changefeed-id custom-changefeed-name pause PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Mon May 6 19:09:29 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.55905592.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3660d0400011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:4289, start at 2024-05-06 19:09:31.555485641 +0800 CST m=+5.268140341 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:31.562 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:31.536 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:31.536 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:32 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfb5738c-4e27-4049-bce2-dcca950da312 {"id":"bfb5738c-4e27-4049-bce2-dcca950da312","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993770} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d97ec75cc bfb5738c-4e27-4049-bce2-dcca950da312 /tidb/cdc/default/default/upstream/7365842097727808488 {"id":7365842097727808488,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfb5738c-4e27-4049-bce2-dcca950da312 {"id":"bfb5738c-4e27-4049-bce2-dcca950da312","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993770} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d97ec75cc bfb5738c-4e27-4049-bce2-dcca950da312 /tidb/cdc/default/default/upstream/7365842097727808488 {"id":7365842097727808488,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfb5738c-4e27-4049-bce2-dcca950da312 {"id":"bfb5738c-4e27-4049-bce2-dcca950da312","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993770} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d97ec75cc bfb5738c-4e27-4049-bce2-dcca950da312 /tidb/cdc/default/default/upstream/7365842097727808488 {"id":7365842097727808488,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5656.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-manager Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365842097727808488,"namespace":"default","id":"ddl-manager","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:09:33.20394584+08:00","start_ts":449575327644778500,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575327644778500,"checkpoint_ts":449575327644778500,"checkpoint_time":"2024-05-06 19:09:33.059"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575326453334022, "checkpoint_time": "2024-05-06 19:09:28.514", "error": null }' + echo '{ "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575326453334022, "checkpoint_time": "2024-05-06 19:09:28.514", "error": null }' { "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575326453334022, "checkpoint_time": "2024-05-06 19:09:28.514", "error": null } ++ echo '{' '"upstream_id":' 7365842025001456188, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449575326453334022, '"checkpoint_time":' '"2024-05-06' '19:09:28.514",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365842025001456188, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449575326453334022, '"checkpoint_time":' '"2024-05-06' '19:09:28.514",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3643.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3660d0400011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:4289, start at 2024-05-06 19:09:31.555485641 +0800 CST m=+5.268140341 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:31.562 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:31.536 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:31.536 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3660d0f40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:4370, start at 2024-05-06 19:09:31.600633836 +0800 CST m=+5.263074608 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:31.606 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:31.581 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:31.581 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0017d9018} {Type:update Path:[Config SyncPointRetention] From: To:0xc0017d9028} {Type:update Path:[Config Consistent] From: To:0xc0014ef7a0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + set +x [Mon May 6 19:09:34 CST 2024] <<<<<< START kafka consumer in ddl_manager case >>>>>> Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365842025001456188,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-27012?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:09:20.645334611+08:00","start_ts":449575322089160705,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":0,"checkpoint_ts":449575326453334022,"checkpoint_time":"2024-05-06 19:09:28.514"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5687.out cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3682.out cli changefeed --changefeed-id custom-changefeed-name resume PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kafka_compression Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x + tso='449575328661372930 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575328661372930 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:09:38 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.57235725.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 26.00 secs (143359539 bytes/sec) [Pipeline] { [Pipeline] cache + set +x wait process 5595 exit for 1-th time... wait process 5595 exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366138440014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:4141, start at 2024-05-06 19:09:38.239562786 +0800 CST m=+5.202910355 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:38.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:38.244 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:38.244 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366138440014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:4141, start at 2024-05-06 19:09:38.239562786 +0800 CST m=+5.202910355 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:38.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:38.244 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:38.244 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366138340015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:4230, start at 2024-05-06 19:09:38.219265531 +0800 CST m=+5.129130380 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:38.226 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:38.189 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:38.189 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process 5595 exit for 3-th time... wait process 5595 exit for 4-th time... wait process 5595 exit for 5-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process 5595 exit for 6-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5625.out cli tso query --pd=http://127.0.0.1:2379 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5595) - No such process wait process 5595 exit for 7-th time... process 5595 already exit [Mon May 6 19:09:41 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.57545756.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:41 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4c3dce4b-0319-40dd-93f4-ee294db0df30 {"id":"4c3dce4b-0319-40dd-93f4-ee294db0df30","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993778} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d980f86d5 4c3dce4b-0319-40dd-93f4-ee294db0df30 /tidb/cdc/default/default/upstream/7365842128837116199 {"id":7365842128837116199,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4c3dce4b-0319-40dd-93f4-ee294db0df30 {"id":"4c3dce4b-0319-40dd-93f4-ee294db0df30","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993778} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d980f86d5 4c3dce4b-0319-40dd-93f4-ee294db0df30 /tidb/cdc/default/default/upstream/7365842128837116199 {"id":7365842128837116199,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4c3dce4b-0319-40dd-93f4-ee294db0df30 {"id":"4c3dce4b-0319-40dd-93f4-ee294db0df30","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993778} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d980f86d5 4c3dce4b-0319-40dd-93f4-ee294db0df30 /tidb/cdc/default/default/upstream/7365842128837116199 {"id":7365842128837116199,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5784.out cli changefeed create --start-ts=449575328661372930 '--sink-uri=kafka://127.0.0.1:9092/ticdc-drop-tables-test-32249?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: d8d33654-f6b6-4f21-8eeb-a84c1dd89973 Info: {"upstream_id":7365842128837116199,"namespace":"default","id":"d8d33654-f6b6-4f21-8eeb-a84c1dd89973","sink_uri":"kafka://127.0.0.1:9092/ticdc-drop-tables-test-32249?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:09:41.96087342+08:00","start_ts":449575328661372930,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575328661372930,"checkpoint_ts":449575328661372930,"checkpoint_time":"2024-05-06 19:09:36.937"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575329861206022, "checkpoint_time": "2024-05-06 19:09:41.514", "error": null }' + echo '{ "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575329861206022, "checkpoint_time": "2024-05-06 19:09:41.514", "error": null }' { "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575329861206022, "checkpoint_time": "2024-05-06 19:09:41.514", "error": null } ++ echo '{' '"upstream_id":' 7365842025001456188, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575329861206022, '"checkpoint_time":' '"2024-05-06' '19:09:41.514",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842025001456188, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575329861206022, '"checkpoint_time":' '"2024-05-06' '19:09:41.514",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3766.out cli changefeed --changefeed-id custom-changefeed-name remove Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449575329861206022 SinkURI: kafka://127.0.0.1:9092/ticdc-cli-test-27012?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + tso='449575329908129793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575329908129793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + set +x [Mon May 6 19:09:43 CST 2024] <<<<<< START kafka consumer in drop_many_tables case >>>>>> [Mon May 6 19:09:43 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.56745676.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:09:41 CST 2024] <<<<<< run test case sink_retry success! >>>>>> table drop_tables.c not exists for 1-th check, retry later + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:46 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80407742-a062-414c-8d48-1a30f74f8f3d {"id":"80407742-a062-414c-8d48-1a30f74f8f3d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d982604e9 80407742-a062-414c-8d48-1a30f74f8f3d /tidb/cdc/default/default/upstream/7365842151430033754 {"id":7365842151430033754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80407742-a062-414c-8d48-1a30f74f8f3d {"id":"80407742-a062-414c-8d48-1a30f74f8f3d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d982604e9 80407742-a062-414c-8d48-1a30f74f8f3d /tidb/cdc/default/default/upstream/7365842151430033754 {"id":7365842151430033754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/80407742-a062-414c-8d48-1a30f74f8f3d {"id":"80407742-a062-414c-8d48-1a30f74f8f3d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d982604e9 80407742-a062-414c-8d48-1a30f74f8f3d /tidb/cdc/default/default/upstream/7365842151430033754 {"id":7365842151430033754,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5733.out cli changefeed create --start-ts=449575329908129793 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-15608?protocol=open-protocol&partition-num=4&kafka-client-id=ddl_puller_lag&kafka-version=2.4.1&max-message-bytes=10485760' table drop_tables.c not exists for 2-th check, retry later Create changefeed successfully! ID: e8d55adc-4f60-4bf4-a52c-5c564b455a36 Info: {"upstream_id":7365842151430033754,"namespace":"default","id":"e8d55adc-4f60-4bf4-a52c-5c564b455a36","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-15608?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=ddl_puller_lag\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:09:47.014441121+08:00","start_ts":449575329908129793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575329908129793,"checkpoint_ts":449575329908129793,"checkpoint_time":"2024-05-06 19:09:41.693"} PASS < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365842097727808488 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:09:33.20394584 +0800 CST StartTs:449575327644778500 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003af19e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575327670992902} {CheckpointTs:449575328063946773 MinTableBarrierTs:449575328063946773 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/def0333d-006f-47e1-9ffd-4d1b8f6ca740 {"id":"def0333d-006f-47e1-9ffd-4d1b8f6ca740","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d97ec785b def0333d-006f-47e1-9ffd-4d1b8f6ca740 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365842097727808488,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:09:33.20394584+08:00","start-ts":449575327644778500,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575327670992902} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575328063946773,"min-table-barrier-ts":449575328063946773,"admin-job-type":0} /tidb/cdc/default/default/task/position/def0333d-006f-47e1-9ffd-4d1b8f6ca740/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842097727808488 {"id":7365842097727808488,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365842097727808488 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:09:33.20394584 +0800 CST StartTs:449575327644778500 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003af19e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575327670992902} {CheckpointTs:449575328063946773 MinTableBarrierTs:449575328063946773 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/def0333d-006f-47e1-9ffd-4d1b8f6ca740 {"id":"def0333d-006f-47e1-9ffd-4d1b8f6ca740","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d97ec785b def0333d-006f-47e1-9ffd-4d1b8f6ca740 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365842097727808488,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:09:33.20394584+08:00","start-ts":449575327644778500,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575327670992902} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575328063946773,"min-table-barrier-ts":449575328063946773,"admin-job-type":0} /tidb/cdc/default/default/task/position/def0333d-006f-47e1-9ffd-4d1b8f6ca740/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842097727808488 + grep -q 'failed to get info:' {"id":7365842097727808488,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365842097727808488 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:09:33.20394584 +0800 CST StartTs:449575327644778500 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003af19e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575327670992902} {CheckpointTs:449575328063946773 MinTableBarrierTs:449575328063946773 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/def0333d-006f-47e1-9ffd-4d1b8f6ca740 {"id":"def0333d-006f-47e1-9ffd-4d1b8f6ca740","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d97ec785b def0333d-006f-47e1-9ffd-4d1b8f6ca740 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365842097727808488,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:09:33.20394584+08:00","start-ts":449575327644778500,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575327670992902} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575328063946773,"min-table-barrier-ts":449575328063946773,"admin-job-type":0} /tidb/cdc/default/default/task/position/def0333d-006f-47e1-9ffd-4d1b8f6ca740/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842097727808488 + grep -q 'etcd info' {"id":7365842097727808488,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Mon May 6 19:09:47 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.58145816.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3864.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-27012?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:48 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365842097727808488 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:09:33.20394584 +0800 CST StartTs:449575327644778500 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003af19e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575327670992902} {CheckpointTs:449575328063946773 MinTableBarrierTs:449575328063946773 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/def0333d-006f-47e1-9ffd-4d1b8f6ca740 {"id":"def0333d-006f-47e1-9ffd-4d1b8f6ca740","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d97ec785b def0333d-006f-47e1-9ffd-4d1b8f6ca740 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365842097727808488,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:09:33.20394584+08:00","start-ts":449575327644778500,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575327670992902} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575328090423305,"min-table-barrier-ts":449575328090423305,"admin-job-type":0} /tidb/cdc/default/default/task/position/def0333d-006f-47e1-9ffd-4d1b8f6ca740/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842097727808488 {"id":7365842097727808488,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365842097727808488 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:09:33.20394584 +0800 CST StartTs:449575327644778500 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003af19e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575327670992902} {CheckpointTs:449575328063946773 MinTableBarrierTs:449575328063946773 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/def0333d-006f-47e1-9ffd-4d1b8f6ca740 {"id":"def0333d-006f-47e1-9ffd-4d1b8f6ca740","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d97ec785b def0333d-006f-47e1-9ffd-4d1b8f6ca740 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365842097727808488,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:09:33.20394584+08:00","start-ts":449575327644778500,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575327670992902} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575328090423305,"min-table-barrier-ts":449575328090423305,"admin-job-type":0} /tidb/cdc/default/default/task/position/def0333d-006f-47e1-9ffd-4d1b8f6ca740/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842097727808488 {"id":7365842097727808488,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365842097727808488 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:09:33.20394584 +0800 CST StartTs:449575327644778500 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003af19e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575327670992902} {CheckpointTs:449575328063946773 MinTableBarrierTs:449575328063946773 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/def0333d-006f-47e1-9ffd-4d1b8f6ca740 {"id":"def0333d-006f-47e1-9ffd-4d1b8f6ca740","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993781} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d97ec785b def0333d-006f-47e1-9ffd-4d1b8f6ca740 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365842097727808488,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-20905?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:09:33.20394584+08:00","start-ts":449575327644778500,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575327670992902} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449575328090423305,"min-table-barrier-ts":449575328090423305,"admin-job-type":0} /tidb/cdc/default/default/task/position/def0333d-006f-47e1-9ffd-4d1b8f6ca740/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842097727808488 {"id":7365842097727808488,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365842025001456188,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-27012?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:09:47.944086759+08:00","start_ts":449575331499343875,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575331499343875,"checkpoint_ts":449575331499343875,"checkpoint_time":"2024-05-06 19:09:47.763"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 19:09:48 CST 2024] <<<<<< START kafka consumer in ddl_puller_lag case >>>>>> table drop_tables.c not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3661c814000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:5767, start at 2024-05-06 19:09:47.40521916 +0800 CST m=+5.151881062 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:47.412 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:47.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:47.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3661c814000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:5767, start at 2024-05-06 19:09:47.40521916 +0800 CST m=+5.151881062 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:47.412 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:47.397 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:47.397 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3661ca340005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:5855, start at 2024-05-06 19:09:47.538285093 +0800 CST m=+5.229246259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:11:47.547 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:09:47.533 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-18:59:47.533 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x table drop_tables.c not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 2-th check, retry later [Mon May 6 19:09:51 CST 2024] <<<<<< START cdc server in kafka_compression case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.71797181.out server --log-file /tmp/tidb_cdc_test/kafka_compression/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_compression/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table drop_tables.c not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 3-th check, retry later + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575332495753217, "checkpoint_time": "2024-05-06 19:09:51.564", "error": null }' + echo '{ "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575332495753217, "checkpoint_time": "2024-05-06 19:09:51.564", "error": null }' { "upstream_id": 7365842025001456188, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575332495753217, "checkpoint_time": "2024-05-06 19:09:51.564", "error": null } ++ echo '{' '"upstream_id":' 7365842025001456188, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575332495753217, '"checkpoint_time":' '"2024-05-06' '19:09:51.564",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842025001456188, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575332495753217, '"checkpoint_time":' '"2024-05-06' '19:09:51.564",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3954.out cli changefeed create --start-ts=449575322089160705 '--sink-uri=kafka://127.0.0.1:9093/ticdc-cli-test-ssl-30088?protocol=open-protocol&ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem&cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem&key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem&kafka-version=2.4.1&max-message-bytes=10485760&insecure-skip-verify=true' --tz=Asia/Shanghai table drop_tables.c exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [WARN] --tz is deprecated in changefeed settings. table ddl_manager.finish_mark not exists for 4-th check, retry later check diff successfully < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:09:54 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01749003-2ff4-4d3e-9781-c017495cd1be {"id":"01749003-2ff4-4d3e-9781-c017495cd1be","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993791} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9849f6cc 01749003-2ff4-4d3e-9781-c017495cd1be /tidb/cdc/default/default/upstream/7365842197538060909 {"id":7365842197538060909,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01749003-2ff4-4d3e-9781-c017495cd1be {"id":"01749003-2ff4-4d3e-9781-c017495cd1be","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993791} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9849f6cc 01749003-2ff4-4d3e-9781-c017495cd1be /tidb/cdc/default/default/upstream/7365842197538060909 {"id":7365842197538060909,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/01749003-2ff4-4d3e-9781-c017495cd1be {"id":"01749003-2ff4-4d3e-9781-c017495cd1be","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993791} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9849f6cc 01749003-2ff4-4d3e-9781-c017495cd1be /tidb/cdc/default/default/upstream/7365842197538060909 {"id":7365842197538060909,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7244.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 1-th time... Create changefeed successfully! ID: 0261b1c6-2f0f-4442-8b82-c93e1eaadd8e Info: {"upstream_id":7365842025001456188,"namespace":"default","id":"0261b1c6-2f0f-4442-8b82-c93e1eaadd8e","sink_uri":"kafka://127.0.0.1:9093/ticdc-cli-test-ssl-30088?protocol=open-protocol\u0026ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem\u0026key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760\u0026insecure-skip-verify=true","create_time":"2024-05-06T19:09:55.196082732+08:00","start_ts":449575322089160705,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575322089160705,"checkpoint_ts":449575322089160705,"checkpoint_time":"2024-05-06 19:09:11.866"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:09:56 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> + set +x + tso='449575333382586370 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575333382586370 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7278.out cli changefeed create --start-ts=449575333382586370 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip' -c gzip Create changefeed successfully! ID: gzip Info: {"upstream_id":7365842197538060909,"namespace":"default","id":"gzip","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=gzip","create_time":"2024-05-06T19:09:57.205759325+08:00","start_ts":449575333382586370,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575333382586370,"checkpoint_ts":449575333382586370,"checkpoint_time":"2024-05-06 19:09:54.947"} PASS + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3973.out cli unsafe delete-service-gc-safepoint coverage: 2.4% of statements in github.com/pingcap/tiflow/... Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 6-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x [Mon May 6 19:09:58 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/06 19:09:57.154 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:57.199 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:57.316 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:57.329 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:58.290 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:58.301 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]: syntax error: operand expected (error token is "[2024/05/06 19:09:57.154 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:57.199 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:57.316 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:57.329 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:58.290 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/06 19:09:58.301 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]") table test.gzip_finish_mark not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3988.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 table ddl_manager.finish_mark not exists for 7-th check, retry later reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table test.gzip_finish_mark not exists for 2-th check, retry later + set +x table ddl_manager.finish_mark not exists for 8-th check, retry later table test.gzip_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7396.out cli changefeed pause -c gzip PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 9-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7430.out cli changefeed remove -c gzip Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4030.out cli unsafe resolve-lock --region=76 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Changefeed remove successfully. ID: gzip CheckpointTs: 449575334378733607 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 10-th check, retry later + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7470.out cli tso query --pd=http://127.0.0.1:2379 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4054.out cli unsafe resolve-lock --region=76 --ts=449575335222050820 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 11-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 26.74 secs (139373206 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed + set +x + tso='449575336672493574 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575336672493574 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7497.out cli changefeed create --start-ts=449575336672493574 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy' -c snappy 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 1519 43 --:--:-- --:--:-- --:--:-- 1514 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: snappy Info: {"upstream_id":7365842197538060909,"namespace":"default","id":"snappy","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=snappy","create_time":"2024-05-06T19:10:09.502810936+08:00","start_ts":449575336672493574,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575336672493574,"checkpoint_ts":449575336672493574,"checkpoint_time":"2024-05-06 19:10:07.497"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 12-th check, retry later + set +x [Mon May 6 19:10:10 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/06 19:10:09.455 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:09.498 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:09.593 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:09.602 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:10.592 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:10.600 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]: syntax error: operand expected (error token is "[2024/05/06 19:10:09.455 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:09.498 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:09.593 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:09.602 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:10.592 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/06 19:10:10.600 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]") table test.snappy_finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36632ba00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:4200, start at 2024-05-06 19:10:10.158733996 +0800 CST m=+5.102814493 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:10.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:10.152 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:10.152 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36632ba00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:4200, start at 2024-05-06 19:10:10.158733996 +0800 CST m=+5.102814493 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:10.164 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:10.152 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:10.152 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36632d240015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:4282, start at 2024-05-06 19:10:10.282359928 +0800 CST m=+5.175861002 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:10.288 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:10.249 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:10.249 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 13-th check, retry later } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 181 100 181 0 0 1028 0 --:--:-- --:--:-- --:--:-- 1034 { "version": "v8.2.0-alpha-25-g58799adc6", "git_hash": "58799adc6338905a3d6177e5829f44667d1aa09e", "id": "266c6a61-689b-432d-961d-41fd0ea5454f", "pid": 3366, "is_owner": true }wait process cdc.test exit for 1-th time... table test.snappy_finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5736.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 14-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:10:14 CST 2024] <<<<<< run test case cli_tls_with_auth success! >>>>>> + set +x + tso='449575338312466433 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575338312466433 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.snappy_finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later ***************** properties ***************** "dotransactions"="false" "insertproportion"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "updateproportion"="0" "requestdistribution"="uniform" "scanproportion"="0" "operationcount"="0" "mysql.db"="changefeed_error" "workload"="core" "recordcount"="20" "readproportion"="0" "mysql.user"="root" "threadcount"="4" "mysql.port"="4000" ********************************************** Run finished, takes 11.104082ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3420.0, Avg(us): 2069, Min(us): 1048, Max(us): 5452, 95th(us): 6000, 99th(us): 6000 [Mon May 6 19:10:16 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57995801.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.snappy_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7611.out cli changefeed pause -c snappy PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 16-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7646.out cli changefeed remove -c snappy + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:19 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c {"id":"8246cc82-5a11-4ba7-954e-e94b9cbee25c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993816} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d61f3 8246cc82-5a11-4ba7-954e-e94b9cbee25c /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c {"id":"8246cc82-5a11-4ba7-954e-e94b9cbee25c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993816} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d61f3 8246cc82-5a11-4ba7-954e-e94b9cbee25c /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c {"id":"8246cc82-5a11-4ba7-954e-e94b9cbee25c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993816} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d61f3 8246cc82-5a11-4ba7-954e-e94b9cbee25c /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5860.out cli changefeed create --start-ts=449575338312466433 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365842285005043972,"namespace":"default","id":"changefeed-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:10:19.875967987+08:00","start_ts":449575338312466433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575338312466433,"checkpoint_ts":449575338312466433,"checkpoint_time":"2024-05-06 19:10:13.753"} PASS Changefeed remove successfully. ID: snappy CheckpointTs: 449575339241504793 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 17-th check, retry later + set +x [Mon May 6 19:10:21 CST 2024] <<<<<< START kafka consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7683.out cli tso query --pd=http://127.0.0.1:2379 + info='{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449575338312466433, "checkpoint_time": "2024-05-06 19:10:13.753", "error": { "time": "2024-05-06T19:10:19.953449671+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575338312466133 is earlier than or equal to GC safepoint at 449575338312466433" } }' + echo '{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449575338312466433, "checkpoint_time": "2024-05-06 19:10:13.753", "error": { "time": "2024-05-06T19:10:19.953449671+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575338312466133 is earlier than or equal to GC safepoint at 449575338312466433" } }' { "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449575338312466433, "checkpoint_time": "2024-05-06 19:10:13.753", "error": { "time": "2024-05-06T19:10:19.953449671+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575338312466133 is earlier than or equal to GC safepoint at 449575338312466433" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449575338312466433, '"checkpoint_time":' '"2024-05-06' '19:10:13.753",' '"error":' '{' '"time":' '"2024-05-06T19:10:19.953449671+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449575338312466133 is earlier than or equal to GC safepoint at '449575338312466433"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449575338312466433, '"checkpoint_time":' '"2024-05-06' '19:10:13.753",' '"error":' '{' '"time":' '"2024-05-06T19:10:19.953449671+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449575338312466133 is earlier than or equal to GC safepoint at '449575338312466433"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575338312466133 is earlier than or equal to GC safepoint at 449575338312466433' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449575338312466133 is earlier than or equal to GC safepoint at 449575338312466433 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5985.out cli changefeed resume -c changefeed-error Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-kmwcn-0kg0j --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "203bff562d6de81cc9265c3fd6a676bd847ba7cb" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-kmwcn" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 18-th check, retry later + set +x + tso='449575340434259973 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575340434259973 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7717.out cli changefeed create --start-ts=449575340434259973 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4' -c lz4 + set +x table changefeed_error.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:10:17 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 1 [Mon May 6 19:10:17 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 2 [Mon May 6 19:10:17 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 3 table test.table1 not exists for 1-th check, retry later table test.table1 not exists for 2-th check, retry later table test.table1 exists table test.table2 exists table test.table3 exists check diff successfully table test.table10 not exists for 1-th check, retry later Create changefeed successfully! ID: lz4 Info: {"upstream_id":7365842197538060909,"namespace":"default","id":"lz4","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=lz4","create_time":"2024-05-06T19:10:23.853914464+08:00","start_ts":449575340434259973,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575340434259973,"checkpoint_ts":449575340434259973,"checkpoint_time":"2024-05-06 19:10:21.847"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.table10 exists table test.table20 exists check diff successfully table ddl_manager.finish_mark not exists for 19-th check, retry later + set +x [Mon May 6 19:10:25 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/06 19:10:23.808 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:23.849 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:23.943 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:23.952 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:24.941 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:24.950 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]: syntax error: operand expected (error token is "[2024/05/06 19:10:23.808 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:23.849 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:23.943 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:23.952 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:24.941 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/06 19:10:24.950 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]") table test.lz4_finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3663f038000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:7049, start at 2024-05-06 19:10:22.752616009 +0800 CST m=+5.412082104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:22.759 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:22.734 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:22.734 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3663f038000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:7049, start at 2024-05-06 19:10:22.752616009 +0800 CST m=+5.412082104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:22.759 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:22.734 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:22.734 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3663f0f80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:7125, start at 2024-05-06 19:10:22.813830434 +0800 CST m=+5.417835660 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:22.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:22.782 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:22.782 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_error.usertable exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.lz4_finish_mark not exists for 2-th check, retry later check diff successfully ***************** properties ***************** "scanproportion"="0" "requestdistribution"="uniform" "recordcount"="20" "operationcount"="0" "mysql.user"="root" "mysql.port"="4000" "workload"="core" "insertproportion"="0" "readallfields"="true" "readproportion"="0" "mysql.db"="changefeed_error" "mysql.host"="127.0.0.1" "dotransactions"="false" "threadcount"="4" "updateproportion"="0" ********************************************** Run finished, takes 4.738805ms INSERT - Takes(s): 0.0, Count: 20, OPS: 6322.9, Avg(us): 836, Min(us): 442, Max(us): 2041, 95th(us): 3000, 99th(us): 3000 check diff successfully {"id":"8246cc82-5a11-4ba7-954e-e94b9cbee25c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993816} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c + [[ /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later [Mon May 6 19:10:27 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.84478449.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 21-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 18.41 secs (202498039 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table test.lz4_finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 22-th check, retry later [Pipeline] sh check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c + [[ /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:30 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47fddb29-09a4-4652-ab59-787a6839d822 {"id":"47fddb29-09a4-4652-ab59-787a6839d822","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993828} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98d111cc 47fddb29-09a4-4652-ab59-787a6839d822 /tidb/cdc/default/default/upstream/7365842347183221608 {"id":7365842347183221608,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47fddb29-09a4-4652-ab59-787a6839d822 {"id":"47fddb29-09a4-4652-ab59-787a6839d822","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993828} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98d111cc 47fddb29-09a4-4652-ab59-787a6839d822 /tidb/cdc/default/default/upstream/7365842347183221608 {"id":7365842347183221608,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/47fddb29-09a4-4652-ab59-787a6839d822 {"id":"47fddb29-09a4-4652-ab59-787a6839d822","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993828} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d98d111cc 47fddb29-09a4-4652-ab59-787a6839d822 /tidb/cdc/default/default/upstream/7365842347183221608 {"id":7365842347183221608,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh [Mon May 6 19:10:31 CST 2024] <<<<<< START kafka consumer in processor_stop_delay case >>>>>> + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 table test.lz4_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7864.out cli changefeed pause -c lz4 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table processor_stop_delay.t not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7901.out cli changefeed remove -c lz4 table processor_stop_delay.t exists check diff failed 1-th time, retry later Changefeed remove successfully. ID: lz4 CheckpointTs: 449575342911520774 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 24-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c + [[ /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/8246cc82-5a11-4ba7-954e-e94b9cbee25c + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7932.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 25-th check, retry later check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449575344182919173 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575344182919173 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7968.out cli changefeed create --start-ts=449575344182919173 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd' -c zstd check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 26-th check, retry later Create changefeed successfully! ID: zstd Info: {"upstream_id":7365842197538060909,"namespace":"default","id":"zstd","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=zstd","create_time":"2024-05-06T19:10:38.14131682+08:00","start_ts":449575344182919173,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575344182919173,"checkpoint_ts":449575344182919173,"checkpoint_time":"2024-05-06 19:10:36.147"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Mon May 6 19:10:39 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/06 19:10:38.107 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:38.137 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:38.242 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:38.251 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:39.242 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:39.250 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]: syntax error: operand expected (error token is "[2024/05/06 19:10:38.107 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:38.137 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:38.242 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:38.251 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:39.242 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/06 19:10:39.250 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]") table test.zstd_finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 27-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 19:10:40 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62456247.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.zstd_finish_mark not exists for 2-th check, retry later Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-174kw Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-b4swd Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 28-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:43 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/310aec67-d772-4e3a-994b-4334c2c835b1 {"id":"310aec67-d772-4e3a-994b-4334c2c835b1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d62e0 310aec67-d772-4e3a-994b-4334c2c835b1 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365842285005043972,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:10:19.875967987+08:00","start-ts":449575338312466433,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T19:10:40.693056278+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575345364140037} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449575341812088837,"min-table-barrier-ts":449575341812088837,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/310aec67-d772-4e3a-994b-4334c2c835b1 {"id":"310aec67-d772-4e3a-994b-4334c2c835b1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d62e0 310aec67-d772-4e3a-994b-4334c2c835b1 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365842285005043972,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:10:19.875967987+08:00","start-ts":449575338312466433,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T19:10:40.693056278+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575345364140037} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449575341812088837,"min-table-barrier-ts":449575341812088837,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/310aec67-d772-4e3a-994b-4334c2c835b1 {"id":"310aec67-d772-4e3a-994b-4334c2c835b1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993840} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d62e0 310aec67-d772-4e3a-994b-4334c2c835b1 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365842285005043972,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:10:19.875967987+08:00","start-ts":449575338312466433,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-06T19:10:40.693056278+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575345364140037} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449575341812088837,"min-table-barrier-ts":449575341812088837,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36650b500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:5150, start at 2024-05-06 19:10:40.879249229 +0800 CST m=+5.470835610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:40.888 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:40.852 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:40.852 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36650b500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:5150, start at 2024-05-06 19:10:40.879249229 +0800 CST m=+5.470835610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:40.888 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:40.852 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:40.852 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36650cd4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:5206, start at 2024-05-06 19:10:40.964412299 +0800 CST m=+5.504931579 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:12:40.971 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:10:40.949 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:00:40.949 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + info='{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449575341812088837, "checkpoint_time": "2024-05-06 19:10:27.103", "error": { "time": "2024-05-06T19:10:40.693056278+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449575341812088837, "checkpoint_time": "2024-05-06 19:10:27.103", "error": { "time": "2024-05-06T19:10:40.693056278+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449575341812088837, "checkpoint_time": "2024-05-06 19:10:27.103", "error": { "time": "2024-05-06T19:10:40.693056278+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575341812088837, '"checkpoint_time":' '"2024-05-06' '19:10:27.103",' '"error":' '{' '"time":' '"2024-05-06T19:10:40.693056278+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575341812088837, '"checkpoint_time":' '"2024-05-06' '19:10:27.103",' '"error":' '{' '"time":' '"2024-05-06T19:10:40.693056278+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6346.out cli changefeed remove -c changefeed-error Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449575341812088837 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 29-th check, retry later table test.zstd_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.8078.out cli changefeed pause -c zstd PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] { [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] checkout The recommended git tool is: git [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // container [Mon May 6 19:10:45 CST 2024] <<<<<< START cdc server in kafka_simple_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.65196521.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] sh wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 30-th check, retry later No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@bc11946; decorates RemoteLauncher[hudson.remoting.Channel@267f1388:JNLP4-connect connection from 10.233.100.189/10.233.100.189:49968] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] sh Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=208378ec-da28-43f7-b744-ed9b153e2caf BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8m3lr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8m3lr pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env check diff failed 5-th time, retry later [Pipeline] sh wait process cdc.test exit for 2-th time... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.8115.out cli changefeed remove -c zstd find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=63b82cca-96ed-4bd2-8307-519d12dbcb1e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-k2r9w GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2 pingcap_tiflow_pull_cdc_integration_kafka_test_1872-k2r9w GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: kafka... <<================= Changefeed remove successfully. ID: zstd CheckpointTs: 449575345126637585 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd PASS [Pipeline] sh The 1 times to try to start tidb cluster... coverage: 2.0% of statements in github.com/pingcap/tiflow/... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=e1d69582-f1a8-4ecd-968f-8c4605a1e437 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8bdr9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9 pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8bdr9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: kafka... <<================= cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 19:10:47 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.64506452.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=b51c37dd-8959-4328-8202-dcfd7468e3ce BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-397v7 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-397v7 pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=ee6dca72-b090-4efb-b99e-26e674b4af9b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-qfjm4 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-qfjm4 pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env table ddl_manager.finish_mark not exists for 31-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: kafka... <<================= + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL check diff failed 6-th time, retry later + set +x * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:48 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad435c99-e72f-4983-9225-26bc89a8a271 {"id":"ad435c99-e72f-4983-9225-26bc89a8a271","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993846} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9917a5ce ad435c99-e72f-4983-9225-26bc89a8a271 /tidb/cdc/default/default/upstream/7365842421338859371 {"id":7365842421338859371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad435c99-e72f-4983-9225-26bc89a8a271 {"id":"ad435c99-e72f-4983-9225-26bc89a8a271","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993846} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9917a5ce ad435c99-e72f-4983-9225-26bc89a8a271 /tidb/cdc/default/default/upstream/7365842421338859371 {"id":7365842421338859371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ad435c99-e72f-4983-9225-26bc89a8a271 {"id":"ad435c99-e72f-4983-9225-26bc89a8a271","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993846} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9917a5ce ad435c99-e72f-4983-9225-26bc89a8a271 /tidb/cdc/default/default/upstream/7365842421338859371 {"id":7365842421338859371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.6567.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-24029?protocol=simple' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/conf/changefeed.toml -c simple-basic + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=131da177-cd62-4918-818d-6722fd3fc7c4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-ds960 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-ds960 pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: kafka... <<================= wait process cdc.test exit for 1-th time... Create changefeed successfully! ID: simple-basic Info: {"upstream_id":7365842421338859371,"namespace":"default","id":"simple-basic","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-24029?protocol=simple","create_time":"2024-05-06T19:10:49.475621429+08:00","start_ts":449575347644792834,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":5,"send_bootstrap_in_msg_count":100,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575347644792834,"checkpoint_ts":449575347644792834,"checkpoint_time":"2024-05-06 19:10:49.353"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) wait process cdc.test exit for 2-th time... [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Pipeline] { [Mon May 6 19:10:50 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:10:50 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/833aec2f-fc50-4c4e-bf35-a922893a32f5 {"id":"833aec2f-fc50-4c4e-bf35-a922893a32f5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993847} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d6336 833aec2f-fc50-4c4e-bf35-a922893a32f5 /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/833aec2f-fc50-4c4e-bf35-a922893a32f5 {"id":"833aec2f-fc50-4c4e-bf35-a922893a32f5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993847} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d6336 833aec2f-fc50-4c4e-bf35-a922893a32f5 /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/833aec2f-fc50-4c4e-bf35-a922893a32f5 {"id":"833aec2f-fc50-4c4e-bf35-a922893a32f5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993847} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d6336 833aec2f-fc50-4c4e-bf35-a922893a32f5 /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6505.out cli changefeed create --start-ts=449575338312466433 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-1 table ddl_manager.finish_mark not exists for 32-th check, retry later Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365842285005043972,"namespace":"default","id":"changefeed-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:10:50.921468645+08:00","start_ts":449575338312466433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575338312466433,"checkpoint_ts":449575338312466433,"checkpoint_time":"2024-05-06 19:10:13.753"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... + set +x start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... The 1 times to try to start tidb cluster... The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:10:52 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 33-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed Verifying downstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 1834 0 --:--:-- --:--:-- --:--:-- 1846 + info='{"state":"warning","resolved_ts":449575338692575259,"checkpoint_ts":449575338692575259,"last_warning":{"time":"2024-05-06T19:10:52.351997379+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449575338692575259,"checkpoint_ts":449575338692575259,"last_warning":{"time":"2024-05-06T19:10:52.351997379+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449575338692575259,"checkpoint_ts":449575338692575259,"last_warning":{"time":"2024-05-06T19:10:52.351997379+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":449575338692575259,"checkpoint_ts":449575338692575259,"last_warning":{"time":"2024-05-06T19:10:52.351997379+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449575338692575259,"checkpoint_ts":449575338692575259,"last_warning":{"time":"2024-05-06T19:10:52.351997379+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6562.out cli changefeed remove -c changefeed-error-1 Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449575338692575259 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 34-th check, retry later Starting Upstream TiDB... start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... [Mon May 6 19:10:54 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 10 [Mon May 6 19:10:54 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 20 [Mon May 6 19:10:54 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: finish table test.finish not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 35-th check, retry later wait process cdc.test exit for 2-th time... [Mon May 6 19:10:56 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic case >>>>>> start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... table test.finish not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Mon May 6 19:10:56 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.66226624.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 36-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish exists check diff successfully Starting Upstream TiDB... wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... table test.finish_mark_for_ddl not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:00 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b878ed44-3b69-4ef9-80ba-4540f7a74a17 {"id":"b878ed44-3b69-4ef9-80ba-4540f7a74a17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993857} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d63b8 b878ed44-3b69-4ef9-80ba-4540f7a74a17 /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b878ed44-3b69-4ef9-80ba-4540f7a74a17 {"id":"b878ed44-3b69-4ef9-80ba-4540f7a74a17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993857} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d63b8 b878ed44-3b69-4ef9-80ba-4540f7a74a17 /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b878ed44-3b69-4ef9-80ba-4540f7a74a17 {"id":"b878ed44-3b69-4ef9-80ba-4540f7a74a17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993857} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d63b8 b878ed44-3b69-4ef9-80ba-4540f7a74a17 /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6675.out cli changefeed create --start-ts=449575338312466433 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-2 table ddl_manager.finish_mark not exists for 37-th check, retry later wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365842285005043972,"namespace":"default","id":"changefeed-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:11:00.488000142+08:00","start_ts":449575338312466433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575338312466433,"checkpoint_ts":449575338312466433,"checkpoint_time":"2024-05-06 19:10:13.753"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:00 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> Starting Upstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark_for_ddl not exists for 2-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449575338312466433, "checkpoint_time": "2024-05-06 19:10:13.753", "error": { "time": "2024-05-06T19:11:00.569117647+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575338312466433 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449575338312466433, "checkpoint_time": "2024-05-06 19:10:13.753", "error": { "time": "2024-05-06T19:11:00.569117647+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575338312466433 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449575338312466433, "checkpoint_time": "2024-05-06 19:10:13.753", "error": { "time": "2024-05-06T19:11:00.569117647+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575338312466433 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575338312466433, '"checkpoint_time":' '"2024-05-06' '19:10:13.753",' '"error":' '{' '"time":' '"2024-05-06T19:11:00.569117647+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575338312466433 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575338312466433, '"checkpoint_time":' '"2024-05-06' '19:10:13.753",' '"error":' '{' '"time":' '"2024-05-06T19:11:00.569117647+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575338312466433 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575338312466433 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449575338312466433 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6760.out cli changefeed remove -c changefeed-error-2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366650ec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:1390, start at 2024-05-06 19:11:01.70176268 +0800 CST m=+5.657963191 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:01.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:01.691 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:01.691 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:11:02 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 38-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36663a4c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:1365, start at 2024-05-06 19:11:00.27256371 +0800 CST m=+5.387944780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:00.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:00.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:00.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36663a4c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:1365, start at 2024-05-06 19:11:00.27256371 +0800 CST m=+5.387944780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:00.279 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:00.243 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:00.243 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36663a380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:1434, start at 2024-05-06 19:11:00.274843488 +0800 CST m=+5.332689771 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:00.284 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:00.288 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:00.288 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449575338312466433 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark_for_ddl not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366650ec0008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:1390, start at 2024-05-06 19:11:01.70176268 +0800 CST m=+5.657963191 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:01.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:01.691 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:01.691 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366651b80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:1458, start at 2024-05-06 19:11:01.778828282 +0800 CST m=+5.656711144 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:01.788 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:01.792 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:01.792 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366652600013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1, pid:1411, start at 2024-05-06 19:11:01.811092689 +0800 CST m=+5.633166785 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:01.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:01.784 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:01.784 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366652600013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1, pid:1411, start at 2024-05-06 19:11:01.811092689 +0800 CST m=+5.633166785 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:01.821 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:01.784 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:01.784 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366655880004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1, pid:1487, start at 2024-05-06 19:11:01.989246823 +0800 CST m=+5.759291262 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:01.997 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:01.986 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:01.986 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 39-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark_for_ddl not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:11:05 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.68066808.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 40-th check, retry later [Mon May 6 19:11:06 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.27842786.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Mon May 6 19:11:06 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.28102812.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark_for_ddl exists + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.6668.out cli changefeed pause -c simple-basic VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366682340012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:1473, start at 2024-05-06 19:11:04.869372611 +0800 CST m=+5.576516296 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:04.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:04.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:04.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366682340012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:1473, start at 2024-05-06 19:11:04.869372611 +0800 CST m=+5.576516296 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:04.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:04.845 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:04.845 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366683d8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:1549, start at 2024-05-06 19:11:04.961652405 +0800 CST m=+5.607489345 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:04.971 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:04.950 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:04.950 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 41-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:09 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10726224-5408-4773-abb9-09cdf6f4458c {"id":"10726224-5408-4773-abb9-09cdf6f4458c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d641c 10726224-5408-4773-abb9-09cdf6f4458c /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10726224-5408-4773-abb9-09cdf6f4458c {"id":"10726224-5408-4773-abb9-09cdf6f4458c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d641c 10726224-5408-4773-abb9-09cdf6f4458c /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/10726224-5408-4773-abb9-09cdf6f4458c {"id":"10726224-5408-4773-abb9-09cdf6f4458c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f4d989d641c 10726224-5408-4773-abb9-09cdf6f4458c /tidb/cdc/default/default/upstream/7365842285005043972 {"id":7365842285005043972,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6860.out cli changefeed create --start-ts=0 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-initialize-error + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666a2980006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5, pid:1366, start at 2024-05-06 19:11:06.925911058 +0800 CST m=+5.612049432 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:06.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:06.918 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:06.918 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666a2980006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5, pid:1366, start at 2024-05-06 19:11:06.925911058 +0800 CST m=+5.612049432 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:06.934 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:06.918 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:06.918 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666a36c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5, pid:1444, start at 2024-05-06 19:11:07.01521282 +0800 CST m=+5.650604769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:07.025 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:07.020 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:07.020 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:09 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26626493-c187-41cc-b725-5e10ddb1ec01 {"id":"26626493-c187-41cc-b725-5e10ddb1ec01","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d996a38ce 26626493-c187-41cc-b725-5e10ddb1ec01 /tidb/cdc/default/default/upstream/7365842510779110247 {"id":7365842510779110247,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26626493-c187-41cc-b725-5e10ddb1ec01 {"id":"26626493-c187-41cc-b725-5e10ddb1ec01","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d996a38ce 26626493-c187-41cc-b725-5e10ddb1ec01 /tidb/cdc/default/default/upstream/7365842510779110247 {"id":7365842510779110247,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/26626493-c187-41cc-b725-5e10ddb1ec01 {"id":"26626493-c187-41cc-b725-5e10ddb1ec01","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d996a38ce 26626493-c187-41cc-b725-5e10ddb1ec01 /tidb/cdc/default/default/upstream/7365842510779110247 {"id":7365842510779110247,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365842285005043972,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:11:09.567727286+08:00","start_ts":449575352900780037,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575352900780037,"checkpoint_ts":449575352900780037,"checkpoint_time":"2024-05-06 19:11:09.403"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:09 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7263c268-1c62-47d6-a2bd-448ee9693883 {"id":"7263c268-1c62-47d6-a2bd-448ee9693883","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99689ece 7263c268-1c62-47d6-a2bd-448ee9693883 /tidb/cdc/default/default/upstream/7365842508760210459 {"id":7365842508760210459,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7263c268-1c62-47d6-a2bd-448ee9693883 {"id":"7263c268-1c62-47d6-a2bd-448ee9693883","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99689ece 7263c268-1c62-47d6-a2bd-448ee9693883 /tidb/cdc/default/default/upstream/7365842508760210459 {"id":7365842508760210459,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7263c268-1c62-47d6-a2bd-448ee9693883 {"id":"7263c268-1c62-47d6-a2bd-448ee9693883","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993866} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99689ece 7263c268-1c62-47d6-a2bd-448ee9693883 /tidb/cdc/default/default/upstream/7365842508760210459 {"id":7365842508760210459,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.6695.out cli changefeed resume -c simple-basic [Mon May 6 19:11:09 CST 2024] <<<<<< START kafka consumer in gc_safepoint case >>>>>> 0 [Mon May 6 19:11:09 CST 2024] <<<<<< START kafka consumer in kv_client_stream_reconnect case >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2889.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 42-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575352900780037, "checkpoint_time": "2024-05-06 19:11:09.403", "error": { "time": "2024-05-06T19:11:09.740669207+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575352900780037, "checkpoint_time": "2024-05-06 19:11:09.403", "error": { "time": "2024-05-06T19:11:09.740669207+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575352900780037, "checkpoint_time": "2024-05-06 19:11:09.403", "error": { "time": "2024-05-06T19:11:09.740669207+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575352900780037, '"checkpoint_time":' '"2024-05-06' '19:11:09.403",' '"error":' '{' '"time":' '"2024-05-06T19:11:09.740669207+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449575353069338625 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575353069338625 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:11:11 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.29252927.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2889.out cli tso query --pd=http://127.0.0.1:2379 + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 43-th check, retry later [Mon May 6 19:11:12 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.29022904.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=kafka + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run kafka + '[' kafka '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Mon May 6 19:11:12 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Mon May 6 19:11:12 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666b1bc000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:1481, start at 2024-05-06 19:11:07.903492369 +0800 CST m=+5.270206636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:07.911 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:07.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:07.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666b1bc000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:1481, start at 2024-05-06 19:11:07.903492369 +0800 CST m=+5.270206636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:07.911 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:07.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:07.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3666b3340015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:1562, start at 2024-05-06 19:11:08.019462669 +0800 CST m=+5.320952901 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:08.027 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:08.031 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:08.031 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + info='{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575352900780037, "checkpoint_time": "2024-05-06 19:11:09.403", "error": { "time": "2024-05-06T19:11:09.740669207+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575352900780037, "checkpoint_time": "2024-05-06 19:11:09.403", "error": { "time": "2024-05-06T19:11:09.740669207+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449575352900780037, "checkpoint_time": "2024-05-06 19:11:09.403", "error": { "time": "2024-05-06T19:11:09.740669207+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449575352900780037, '"checkpoint_time":' '"2024-05-06' '19:11:09.403",' '"error":' '{' '"time":' '"2024-05-06T19:11:09.740669207+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later + set +x + tso='449575353599393793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575353599393793 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:11:13 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.29202922.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365842510779110247 449575353867829253 449575352963170315 table ddl_manager.finish_mark not exists for 44-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:14 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2d5ad12-29d7-4a72-80cd-7f6c380a099a {"id":"d2d5ad12-29d7-4a72-80cd-7f6c380a099a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993871} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9978b0d5 d2d5ad12-29d7-4a72-80cd-7f6c380a099a /tidb/cdc/default/default/upstream/7365842522560430141 {"id":7365842522560430141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2d5ad12-29d7-4a72-80cd-7f6c380a099a {"id":"d2d5ad12-29d7-4a72-80cd-7f6c380a099a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993871} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9978b0d5 d2d5ad12-29d7-4a72-80cd-7f6c380a099a /tidb/cdc/default/default/upstream/7365842522560430141 {"id":7365842522560430141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2d5ad12-29d7-4a72-80cd-7f6c380a099a {"id":"d2d5ad12-29d7-4a72-80cd-7f6c380a099a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993871} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9978b0d5 d2d5ad12-29d7-4a72-80cd-7f6c380a099a /tidb/cdc/default/default/upstream/7365842522560430141 {"id":7365842522560430141,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2993.out cli changefeed create --start-ts=449575353069338625 '--sink-uri=kafka://127.0.0.1:9092/ticdc-default-value-test-18841?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2910.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: eab5008b-2cdc-4b18-97f0-316e68d9c3c8 Info: {"upstream_id":7365842522560430141,"namespace":"default","id":"eab5008b-2cdc-4b18-97f0-316e68d9c3c8","sink_uri":"kafka://127.0.0.1:9092/ticdc-default-value-test-18841?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:11:15.235512316+08:00","start_ts":449575353069338625,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575353069338625,"checkpoint_ts":449575353069338625,"checkpoint_time":"2024-05-06 19:11:10.046"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... run task successfully check_changefeed_state http://127.0.0.1:2379 0e65ce3d-1a9d-45fd-a04a-2d733548fe91 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=0e65ce3d-1a9d-45fd-a04a-2d733548fe91 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 0e65ce3d-1a9d-45fd-a04a-2d733548fe91 -s + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:15 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87d24618-0d79-40f9-83d3-979736c8da7d {"id":"87d24618-0d79-40f9-83d3-979736c8da7d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993872} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d996b9a81 87d24618-0d79-40f9-83d3-979736c8da7d /tidb/cdc/default/default/upstream/7365842503057228906 {"id":7365842503057228906,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87d24618-0d79-40f9-83d3-979736c8da7d {"id":"87d24618-0d79-40f9-83d3-979736c8da7d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993872} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d996b9a81 87d24618-0d79-40f9-83d3-979736c8da7d /tidb/cdc/default/default/upstream/7365842503057228906 {"id":7365842503057228906,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87d24618-0d79-40f9-83d3-979736c8da7d {"id":"87d24618-0d79-40f9-83d3-979736c8da7d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993872} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d996b9a81 87d24618-0d79-40f9-83d3-979736c8da7d /tidb/cdc/default/default/upstream/7365842503057228906 {"id":7365842503057228906,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:11:15 CST 2024] <<<<<< START kafka consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 d30b6f5b-6311-4ce7-b167-bf583703931d normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=d30b6f5b-6311-4ce7-b167-bf583703931d + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c d30b6f5b-6311-4ce7-b167-bf583703931d -s start tidb cluster in /tmp/tidb_cdc_test/kafka_sink_error_resume Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + info='{ "upstream_id": 7365842510779110247, "namespace": "default", "id": "0e65ce3d-1a9d-45fd-a04a-2d733548fe91", "state": "stopped", "checkpoint_tso": 449575354391855110, "checkpoint_time": "2024-05-06 19:11:15.091", "error": null }' + echo '{ "upstream_id": 7365842510779110247, "namespace": "default", "id": "0e65ce3d-1a9d-45fd-a04a-2d733548fe91", "state": "stopped", "checkpoint_tso": 449575354391855110, "checkpoint_time": "2024-05-06 19:11:15.091", "error": null }' { "upstream_id": 7365842510779110247, "namespace": "default", "id": "0e65ce3d-1a9d-45fd-a04a-2d733548fe91", "state": "stopped", "checkpoint_tso": 449575354391855110, "checkpoint_time": "2024-05-06 19:11:15.091", "error": null } ++ echo '{' '"upstream_id":' 7365842510779110247, '"namespace":' '"default",' '"id":' '"0e65ce3d-1a9d-45fd-a04a-2d733548fe91",' '"state":' '"stopped",' '"checkpoint_tso":' 449575354391855110, '"checkpoint_time":' '"2024-05-06' '19:11:15.091",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365842510779110247, '"namespace":' '"default",' '"id":' '"0e65ce3d-1a9d-45fd-a04a-2d733548fe91",' '"state":' '"stopped",' '"checkpoint_tso":' 449575354391855110, '"checkpoint_time":' '"2024-05-06' '19:11:15.091",' '"error":' null '}' ++ jq -r .error.message + info='{ "upstream_id": 7365842503057228906, "namespace": "default", "id": "d30b6f5b-6311-4ce7-b167-bf583703931d", "state": "normal", "checkpoint_tso": 449575354457915398, "checkpoint_time": "2024-05-06 19:11:15.343", "error": null }' + echo '{ "upstream_id": 7365842503057228906, "namespace": "default", "id": "d30b6f5b-6311-4ce7-b167-bf583703931d", "state": "normal", "checkpoint_tso": 449575354457915398, "checkpoint_time": "2024-05-06 19:11:15.343", "error": null }' { "upstream_id": 7365842503057228906, "namespace": "default", "id": "d30b6f5b-6311-4ce7-b167-bf583703931d", "state": "normal", "checkpoint_tso": 449575354457915398, "checkpoint_time": "2024-05-06 19:11:15.343", "error": null } ++ echo '{' '"upstream_id":' 7365842503057228906, '"namespace":' '"default",' '"id":' '"d30b6f5b-6311-4ce7-b167-bf583703931d",' '"state":' '"normal",' '"checkpoint_tso":' 449575354457915398, '"checkpoint_time":' '"2024-05-06' '19:11:15.343",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842503057228906, '"namespace":' '"default",' '"id":' '"d30b6f5b-6311-4ce7-b167-bf583703931d",' '"state":' '"normal",' '"checkpoint_tso":' 449575354457915398, '"checkpoint_time":' '"2024-05-06' '19:11:15.343",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365842510779110247 check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:11:15 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> table ddl_manager.finish_mark not exists for 45-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + set +x [Mon May 6 19:11:16 CST 2024] <<<<<< START kafka consumer in default_value case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/time v0.5.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:16 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/574868ad-9d1a-4ccc-851b-8f119a3239a5 {"id":"574868ad-9d1a-4ccc-851b-8f119a3239a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993874} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d997ab9dd 574868ad-9d1a-4ccc-851b-8f119a3239a5 /tidb/cdc/default/default/upstream/7365842521824765850 {"id":7365842521824765850,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/574868ad-9d1a-4ccc-851b-8f119a3239a5 {"id":"574868ad-9d1a-4ccc-851b-8f119a3239a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993874} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d997ab9dd 574868ad-9d1a-4ccc-851b-8f119a3239a5 /tidb/cdc/default/default/upstream/7365842521824765850 {"id":7365842521824765850,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/574868ad-9d1a-4ccc-851b-8f119a3239a5 {"id":"574868ad-9d1a-4ccc-851b-8f119a3239a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993874} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d997ab9dd 574868ad-9d1a-4ccc-851b-8f119a3239a5 /tidb/cdc/default/default/upstream/7365842521824765850 {"id":7365842521824765850,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2969.out cli changefeed create --start-ts=449575353599393793 '--sink-uri=kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-19872?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda + set +x + tso='449575354495401985 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575354495401985 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:11:16 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.29442946.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading golang.org/x/text v0.14.0 Create changefeed successfully! ID: e7743732-32b3-439e-865d-9eef11e4f9d4 Info: {"upstream_id":7365842521824765850,"namespace":"default","id":"e7743732-32b3-439e-865d-9eef11e4f9d4","sink_uri":"kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-19872?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:11:17.297779193+08:00","start_ts":449575353599393793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575353599393793,"checkpoint_ts":449575353599393793,"checkpoint_time":"2024-05-06 19:11:12.068"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575354932133891, "checkpoint_time": "2024-05-06 19:11:17.152", "error": null }' + echo '{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575354932133891, "checkpoint_time": "2024-05-06 19:11:17.152", "error": null }' { "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575354932133891, "checkpoint_time": "2024-05-06 19:11:17.152", "error": null } ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575354932133891, '"checkpoint_time":' '"2024-05-06' '19:11:17.152",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575354932133891, '"checkpoint_time":' '"2024-05-06' '19:11:17.152",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7058.out cli changefeed pause -c changefeed-initialize-error ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 46-th check, retry later check diff failed 2-th time, retry later + set +x [Mon May 6 19:11:18 CST 2024] <<<<<< START kafka consumer in many_pk_or_uk case >>>>>> go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading google.golang.org/api v0.170.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading cloud.google.com/go v0.112.2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 run task successfully go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check_changefeed_state http://127.0.0.1:2379 0e65ce3d-1a9d-45fd-a04a-2d733548fe91 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=0e65ce3d-1a9d-45fd-a04a-2d733548fe91 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 0e65ce3d-1a9d-45fd-a04a-2d733548fe91 -s + info='{ "upstream_id": 7365842510779110247, "namespace": "default", "id": "0e65ce3d-1a9d-45fd-a04a-2d733548fe91", "state": "normal", "checkpoint_tso": 449575354391855110, "checkpoint_time": "2024-05-06 19:11:15.091", "error": null }' + echo '{ "upstream_id": 7365842510779110247, "namespace": "default", "id": "0e65ce3d-1a9d-45fd-a04a-2d733548fe91", "state": "normal", "checkpoint_tso": 449575354391855110, "checkpoint_time": "2024-05-06 19:11:15.091", "error": null }' { "upstream_id": 7365842510779110247, "namespace": "default", "id": "0e65ce3d-1a9d-45fd-a04a-2d733548fe91", "state": "normal", "checkpoint_tso": 449575354391855110, "checkpoint_time": "2024-05-06 19:11:15.091", "error": null } ++ echo '{' '"upstream_id":' 7365842510779110247, '"namespace":' '"default",' '"id":' '"0e65ce3d-1a9d-45fd-a04a-2d733548fe91",' '"state":' '"normal",' '"checkpoint_tso":' 449575354391855110, '"checkpoint_time":' '"2024-05-06' '19:11:15.091",' '"error":' null '}' ++ jq -r .state go: downloading golang.org/x/text v0.14.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:11:19 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + info='{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449575354932133891, "checkpoint_time": "2024-05-06 19:11:17.152", "error": { "time": "2024-05-06T19:11:09.740669207+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449575354932133891, "checkpoint_time": "2024-05-06 19:11:17.152", "error": { "time": "2024-05-06T19:11:09.740669207+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449575354932133891, "checkpoint_time": "2024-05-06 19:11:17.152", "error": { "time": "2024-05-06T19:11:09.740669207+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449575354932133891, '"checkpoint_time":' '"2024-05-06' '19:11:17.152",' '"error":' '{' '"time":' '"2024-05-06T19:11:09.740669207+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449575354932133891, '"checkpoint_time":' '"2024-05-06' '19:11:17.152",' '"error":' '{' '"time":' '"2024-05-06T19:11:09.740669207+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7154.out cli changefeed resume -c changefeed-initialize-error + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842510779110247, '"namespace":' '"default",' '"id":' '"0e65ce3d-1a9d-45fd-a04a-2d733548fe91",' '"state":' '"normal",' '"checkpoint_tso":' 449575354391855110, '"checkpoint_time":' '"2024-05-06' '19:11:15.091",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365842510779110247 449575354391855109 449575354391855110 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:20 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/745fba00-38cc-42bc-a394-de0c8d291cd3 {"id":"745fba00-38cc-42bc-a394-de0c8d291cd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993877} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d998010d8 745fba00-38cc-42bc-a394-de0c8d291cd3 /tidb/cdc/default/default/upstream/7365842539599567596 {"id":7365842539599567596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/745fba00-38cc-42bc-a394-de0c8d291cd3 {"id":"745fba00-38cc-42bc-a394-de0c8d291cd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993877} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d998010d8 745fba00-38cc-42bc-a394-de0c8d291cd3 /tidb/cdc/default/default/upstream/7365842539599567596 {"id":7365842539599567596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/745fba00-38cc-42bc-a394-de0c8d291cd3 {"id":"745fba00-38cc-42bc-a394-de0c8d291cd3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993877} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d998010d8 745fba00-38cc-42bc-a394-de0c8d291cd3 /tidb/cdc/default/default/upstream/7365842539599567596 {"id":7365842539599567596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2999.out cli changefeed create --start-ts=449575354495401985 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resolve-lock-test-29515?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' check diff successfully Create changefeed successfully! ID: 904c2c08-c052-47ef-a28f-003b26390a46 Info: {"upstream_id":7365842539599567596,"namespace":"default","id":"904c2c08-c052-47ef-a28f-003b26390a46","sink_uri":"kafka://127.0.0.1:9092/ticdc-resolve-lock-test-29515?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:11:20.558693239+08:00","start_ts":449575354495401985,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575354495401985,"checkpoint_ts":449575354495401985,"checkpoint_time":"2024-05-06 19:11:15.486"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366749840004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:10155, start at 2024-05-06 19:11:17.605036421 +0800 CST m=+5.553744265 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:17.615 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:17.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:17.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366749840004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:10155, start at 2024-05-06 19:11:17.605036421 +0800 CST m=+5.553744265 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:17.615 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:17.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:17.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667489c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-hfkv1-wxp6j, pid:10232, start at 2024-05-06 19:11:17.587496396 +0800 CST m=+5.485688849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:17.594 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:17.593 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:17.593 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... run task successfully table ddl_manager.finish_mark not exists for 47-th check, retry later check_changefeed_state http://127.0.0.1:2379 0e65ce3d-1a9d-45fd-a04a-2d733548fe91 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=0e65ce3d-1a9d-45fd-a04a-2d733548fe91 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 0e65ce3d-1a9d-45fd-a04a-2d733548fe91 -s wait process cdc.test exit for 1-th time... + info='{ "upstream_id": 7365842510779110247, "namespace": "default", "id": "0e65ce3d-1a9d-45fd-a04a-2d733548fe91", "state": "stopped", "checkpoint_tso": 449575354391855110, "checkpoint_time": "2024-05-06 19:11:15.091", "error": null }' + echo '{ "upstream_id": 7365842510779110247, "namespace": "default", "id": "0e65ce3d-1a9d-45fd-a04a-2d733548fe91", "state": "stopped", "checkpoint_tso": 449575354391855110, "checkpoint_time": "2024-05-06 19:11:15.091", "error": null }' { "upstream_id": 7365842510779110247, "namespace": "default", "id": "0e65ce3d-1a9d-45fd-a04a-2d733548fe91", "state": "stopped", "checkpoint_tso": 449575354391855110, "checkpoint_time": "2024-05-06 19:11:15.091", "error": null } ++ echo '{' '"upstream_id":' 7365842510779110247, '"namespace":' '"default",' '"id":' '"0e65ce3d-1a9d-45fd-a04a-2d733548fe91",' '"state":' '"stopped",' '"checkpoint_tso":' 449575354391855110, '"checkpoint_time":' '"2024-05-06' '19:11:15.091",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365842510779110247, '"namespace":' '"default",' '"id":' '"0e65ce3d-1a9d-45fd-a04a-2d733548fe91",' '"state":' '"stopped",' '"checkpoint_tso":' 449575354391855110, '"checkpoint_time":' '"2024-05-06' '19:11:15.091",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 2-th time... go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/otiai10/copy v1.2.0 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 check_changefeed_state http://127.0.0.1:2379 1b4d5d4c-20b6-47b6-994a-dfcff05efa08 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=1b4d5d4c-20b6-47b6-994a-dfcff05efa08 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 1b4d5d4c-20b6-47b6-994a-dfcff05efa08 -s Starting Upstream TiDB... go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da + info='{ "upstream_id": 7365842510779110247, "namespace": "default", "id": "1b4d5d4c-20b6-47b6-994a-dfcff05efa08", "state": "normal", "checkpoint_tso": 449575356056469511, "checkpoint_time": "2024-05-06 19:11:21.441", "error": null }' + echo '{ "upstream_id": 7365842510779110247, "namespace": "default", "id": "1b4d5d4c-20b6-47b6-994a-dfcff05efa08", "state": "normal", "checkpoint_tso": 449575356056469511, "checkpoint_time": "2024-05-06 19:11:21.441", "error": null }' { "upstream_id": 7365842510779110247, "namespace": "default", "id": "1b4d5d4c-20b6-47b6-994a-dfcff05efa08", "state": "normal", "checkpoint_tso": 449575356056469511, "checkpoint_time": "2024-05-06 19:11:21.441", "error": null } ++ echo '{' '"upstream_id":' 7365842510779110247, '"namespace":' '"default",' '"id":' '"1b4d5d4c-20b6-47b6-994a-dfcff05efa08",' '"state":' '"normal",' '"checkpoint_tso":' 449575356056469511, '"checkpoint_time":' '"2024-05-06' '19:11:21.441",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842510779110247, '"namespace":' '"default",' '"id":' '"1b4d5d4c-20b6-47b6-994a-dfcff05efa08",' '"state":' '"normal",' '"checkpoint_tso":' 449575356056469511, '"checkpoint_time":' '"2024-05-06' '19:11:21.441",' '"error":' null '}' ++ jq -r .error.message cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:11:21 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365842510779110247 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575354932133891, "checkpoint_time": "2024-05-06 19:11:17.152", "error": null }' + echo '{ "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575354932133891, "checkpoint_time": "2024-05-06 19:11:17.152", "error": null }' { "upstream_id": 7365842285005043972, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449575354932133891, "checkpoint_time": "2024-05-06 19:11:17.152", "error": null } ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575354932133891, '"checkpoint_time":' '"2024-05-06' '19:11:17.152",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842285005043972, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449575354932133891, '"checkpoint_time":' '"2024-05-06' '19:11:17.152",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7238.out cli changefeed remove -c changefeed-initialize-error go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 + set +x [Mon May 6 19:11:22 CST 2024] <<<<<< START kafka consumer in resolve_lock case >>>>>> go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/google/btree v1.1.2 go: downloading google.golang.org/protobuf v1.33.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11570.out cli tso query --pd=http://127.0.0.1:2379 go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda table ddl_manager.finish_mark not exists for 48-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:11:22 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> go: downloading golang.org/x/text v0.14.0 Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449575354932133891 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-8216?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 30.16 secs (123605747 bytes/sec) [Pipeline] { go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + set +x + tso='449575356387033089 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575356387033089 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:11:24 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1161011612.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table ddl_manager.finish_mark not exists for 49-th check, retry later go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/spkg/bom v1.0.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading google.golang.org/api v0.170.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/dolthub/maphash v0.1.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible wait process cdc.test exit for 1-th time... go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 run task successfully go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading k8s.io/apimachinery v0.28.6 wait process cdc.test exit for 2-th time... go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 Changefeed remove successfully. ID: 0e65ce3d-1a9d-45fd-a04a-2d733548fe91 CheckpointTs: 449575354391855110 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-32468?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_forward http://127.0.0.1:2379 7365842510779110247 449575354391855109 449575354391855110 449575356056469511 go: downloading github.com/google/gofuzz v1.2.0 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:11:25 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully table test.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 50-th check, retry later go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Changefeed remove successfully. ID: 1b4d5d4c-20b6-47b6-994a-dfcff05efa08 CheckpointTs: 449575357275439111 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-32468?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_cleared http://127.0.0.1:2379 7365842510779110247 run task successfully \033[0;36m<<< Run all test success >>>\033[0m table test.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:27 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f47bcd22-0a81-4c14-bae7-f80fccd057f7 {"id":"f47bcd22-0a81-4c14-bae7-f80fccd057f7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99aab6d1 f47bcd22-0a81-4c14-bae7-f80fccd057f7 /tidb/cdc/default/default/upstream/7365842579283609594 {"id":7365842579283609594,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f47bcd22-0a81-4c14-bae7-f80fccd057f7 {"id":"f47bcd22-0a81-4c14-bae7-f80fccd057f7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99aab6d1 f47bcd22-0a81-4c14-bae7-f80fccd057f7 /tidb/cdc/default/default/upstream/7365842579283609594 {"id":7365842579283609594,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f47bcd22-0a81-4c14-bae7-f80fccd057f7 {"id":"f47bcd22-0a81-4c14-bae7-f80fccd057f7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993884} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99aab6d1 f47bcd22-0a81-4c14-bae7-f80fccd057f7 /tidb/cdc/default/default/upstream/7365842579283609594 {"id":7365842579283609594,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11675.out cli changefeed create --start-ts=449575356387033089 '--sink-uri=kafka://127.0.0.1:9092/ticdc-partition-table-test-503?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 99822451-9f9b-4746-9e41-9929720fc6c4 Info: {"upstream_id":7365842579283609594,"namespace":"default","id":"99822451-9f9b-4746-9e41-9929720fc6c4","sink_uri":"kafka://127.0.0.1:9092/ticdc-partition-table-test-503?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:11:27.776194346+08:00","start_ts":449575356387033089,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575356387033089,"checkpoint_ts":449575356387033089,"checkpoint_time":"2024-05-06 19:11:22.702"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 51-th check, retry later wait process cdc.test exit for 3-th time... go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:29 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> + set +x [Mon May 6 19:11:29 CST 2024] <<<<<< START kafka consumer in partition_table case >>>>>> table test.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667e7cc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:9340, start at 2024-05-06 19:11:27.763995485 +0800 CST m=+5.933070168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:27.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:27.731 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:27.731 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667e7cc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:9340, start at 2024-05-06 19:11:27.763995485 +0800 CST m=+5.933070168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:27.772 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:27.731 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:27.731 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3667ea200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:9403, start at 2024-05-06 19:11:27.902309842 +0800 CST m=+6.002075486 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:27.910 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:27.880 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:27.880 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 52-th check, retry later table test.finish_mark not exists for 4-th check, retry later [Pipeline] } [Mon May 6 19:11:32 CST 2024] <<<<<< START cdc server in kafka_sink_error_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=1*return(true)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_sink_error_resume.1073310735.out server --log-file /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 53-th check, retry later table test.finish_mark not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] container [Pipeline] { [Pipeline] // cache [Pipeline] } [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh table ddl_manager.finish_mark not exists for 54-th check, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table test.finish_mark not exists for 6-th check, retry later < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:35 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e98ffbc0-02e8-4a63-87c9-75745f4c825f {"id":"e98ffbc0-02e8-4a63-87c9-75745f4c825f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993892} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cb03cc e98ffbc0-02e8-4a63-87c9-75745f4c825f /tidb/cdc/default/default/upstream/7365842619117964968 {"id":7365842619117964968,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e98ffbc0-02e8-4a63-87c9-75745f4c825f {"id":"e98ffbc0-02e8-4a63-87c9-75745f4c825f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993892} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cb03cc e98ffbc0-02e8-4a63-87c9-75745f4c825f /tidb/cdc/default/default/upstream/7365842619117964968 {"id":7365842619117964968,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e98ffbc0-02e8-4a63-87c9-75745f4c825f {"id":"e98ffbc0-02e8-4a63-87c9-75745f4c825f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993892} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d99cb03cc e98ffbc0-02e8-4a63-87c9-75745f4c825f /tidb/cdc/default/default/upstream/7365842619117964968 {"id":7365842619117964968,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:11:35 CST 2024] <<<<<< START kafka consumer in kafka_sink_error_resume case >>>>>> [Pipeline] // dir [Pipeline] } table ddl_manager.finish_mark not exists for 55-th check, retry later [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // container [Pipeline] sh table test.finish_mark not exists for 7-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 check_changefeed_status 127.0.0.1:8300 1f82ecb1-be26-4acd-ab0f-5ead830f7d54 warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=1f82ecb1-be26-4acd-ab0f-5ead830f7d54 + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/1f82ecb1-be26-4acd-ab0f-5ead830f7d54/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 438 0 --:--:-- --:--:-- --:--:-- 438 + info='{"state":"normal","resolved_ts":449575359920734244,"checkpoint_ts":449575359920734244}' + echo '{"state":"normal","resolved_ts":449575359920734244,"checkpoint_ts":449575359920734244}' {"state":"normal","resolved_ts":449575359920734244,"checkpoint_ts":449575359920734244} ++ echo '{"state":"normal","resolved_ts":449575359920734244,"checkpoint_ts":449575359920734244}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later [Pipeline] // stage Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=486ba5b8-64f3-42f2-917f-df0a06f757eb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-kmwcn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514 pingcap_tiflow_pull_cdc_integration_kafka_test_1872-kmwcn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } table ddl_manager.finish_mark not exists for 56-th check, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table test.finish_mark not exists for 8-th check, retry later check_changefeed_status 127.0.0.1:8300 1f82ecb1-be26-4acd-ab0f-5ead830f7d54 warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=1f82ecb1-be26-4acd-ab0f-5ead830f7d54 + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/1f82ecb1-be26-4acd-ab0f-5ead830f7d54/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 569 0 --:--:-- --:--:-- --:--:-- 573 + info='{"state":"normal","resolved_ts":449575360628523012,"checkpoint_ts":449575360064651298}' + echo '{"state":"normal","resolved_ts":449575360628523012,"checkpoint_ts":449575360064651298}' {"state":"normal","resolved_ts":449575360628523012,"checkpoint_ts":449575360064651298} ++ echo '{"state":"normal","resolved_ts":449575360628523012,"checkpoint_ts":449575360064651298}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... table ddl_manager.finish_mark not exists for 57-th check, retry later Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 9-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 58-th check, retry later table test.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 1f82ecb1-be26-4acd-ab0f-5ead830f7d54 warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=1f82ecb1-be26-4acd-ab0f-5ead830f7d54 + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/1f82ecb1-be26-4acd-ab0f-5ead830f7d54/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 1636 0 --:--:-- --:--:-- --:--:-- 1648 + info='{"state":"warning","resolved_ts":449575361676836866,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449575361676836866,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449575361676836866,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449575361676836866,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":449575361676836866,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .last_warning.message + error_msg='kafka sink injected error' + [[ ! kafka sink injected error =~ kafka ]] run task successfully check_changefeed_status 127.0.0.1:8300 1f82ecb1-be26-4acd-ab0f-5ead830f7d54 normal + endpoint=127.0.0.1:8300 + changefeed_id=1f82ecb1-be26-4acd-ab0f-5ead830f7d54 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/1f82ecb1-be26-4acd-ab0f-5ead830f7d54/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 1849 0 --:--:-- --:--:-- --:--:-- 1848 100 244 100 244 0 0 1847 0 --:--:-- --:--:-- --:--:-- 1848 + info='{"state":"warning","resolved_ts":449575361938980868,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449575361938980868,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449575361938980868,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449575361938980868,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 59-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_changefeed_status 127.0.0.1:8300 1f82ecb1-be26-4acd-ab0f-5ead830f7d54 normal + endpoint=127.0.0.1:8300 + changefeed_id=1f82ecb1-be26-4acd-ab0f-5ead830f7d54 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/1f82ecb1-be26-4acd-ab0f-5ead830f7d54/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table ddl_manager.finish_mark not exists for 60-th check, retry later Verifying downstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 1611 0 --:--:-- --:--:-- --:--:-- 1615 + info='{"state":"warning","resolved_ts":449575362463268869,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449575362463268869,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449575362463268869,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449575362463268869,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 61-th check, retry later check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 2-th check, retry later check diff successfully Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366933c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:4332, start at 2024-05-06 19:11:49.014388482 +0800 CST m=+5.351099115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:49.020 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:48.978 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:48.978 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... table partition_table.finish_mark not exists for 3-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:11:50 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> check_changefeed_status 127.0.0.1:8300 1f82ecb1-be26-4acd-ab0f-5ead830f7d54 normal + endpoint=127.0.0.1:8300 + changefeed_id=1f82ecb1-be26-4acd-ab0f-5ead830f7d54 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/1f82ecb1-be26-4acd-ab0f-5ead830f7d54/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 1678 0 --:--:-- --:--:-- --:--:-- 1682 + info='{"state":"warning","resolved_ts":449575363511844868,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449575363511844868,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449575363511844868,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449575363511844868,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 3-th time, retry later table ddl_manager.finish_mark not exists for 62-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366933c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:4332, start at 2024-05-06 19:11:49.014388482 +0800 CST m=+5.351099115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:49.020 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:48.978 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:48.978 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669353c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:4415, start at 2024-05-06 19:11:49.109135396 +0800 CST m=+5.393290344 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:49.117 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:49.121 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:49.121 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 63-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:11:53 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.57295731.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36697f70000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514, pid:1284, start at 2024-05-06 19:11:53.831365935 +0800 CST m=+5.856736993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:53.840 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:53.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:53.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 64-th check, retry later table partition_table.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36697f70000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514, pid:1284, start at 2024-05-06 19:11:53.831365935 +0800 CST m=+5.856736993 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:53.840 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:53.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:53.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366980800011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514, pid:1363, start at 2024-05-06 19:11:53.910377929 +0800 CST m=+5.875704216 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:53.917 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:53.888 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:53.888 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 65-th check, retry later table partition_table.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:11:57 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc62d804-f512-42fa-9404-dfb69246b62b {"id":"fc62d804-f512-42fa-9404-dfb69246b62b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a2067cd fc62d804-f512-42fa-9404-dfb69246b62b /tidb/cdc/default/default/upstream/7365842716379252717 {"id":7365842716379252717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc62d804-f512-42fa-9404-dfb69246b62b {"id":"fc62d804-f512-42fa-9404-dfb69246b62b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a2067cd fc62d804-f512-42fa-9404-dfb69246b62b /tidb/cdc/default/default/upstream/7365842716379252717 {"id":7365842716379252717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc62d804-f512-42fa-9404-dfb69246b62b {"id":"fc62d804-f512-42fa-9404-dfb69246b62b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993914} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a2067cd fc62d804-f512-42fa-9404-dfb69246b62b /tidb/cdc/default/default/upstream/7365842716379252717 {"id":7365842716379252717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:11:57 CST 2024] <<<<<< START kafka consumer in changefeed_reconstruct case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "mysql.db"="changefeed_reconstruct" "workload"="core" "dotransactions"="false" "mysql.port"="4000" "updateproportion"="0" "operationcount"="0" "readallfields"="true" "mysql.host"="127.0.0.1" "insertproportion"="0" "readproportion"="0" "threadcount"="4" "scanproportion"="0" "recordcount"="50" "requestdistribution"="uniform" "mysql.user"="root" ********************************************** Run finished, takes 23.884212ms INSERT - Takes(s): 0.0, Count: 46, OPS: 2357.6, Avg(us): 1640, Min(us): 1024, Max(us): 5420, 95th(us): 5000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 1f82ecb1-be26-4acd-ab0f-5ead830f7d54 normal + endpoint=127.0.0.1:8300 + changefeed_id=1f82ecb1-be26-4acd-ab0f-5ead830f7d54 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/1f82ecb1-be26-4acd-ab0f-5ead830f7d54/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 1538 0 --:--:-- --:--:-- --:--:-- 1544 + info='{"state":"warning","resolved_ts":449575365346852871,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":449575365346852871,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":449575365346852871,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":449575365346852871,"checkpoint_ts":449575360064651298,"last_warning":{"time":"2024-05-06T19:11:39.598510875+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 4-th time, retry later [Mon May 6 19:11:58 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.28102812.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 66-th check, retry later table partition_table.finish_mark not exists for 7-th check, retry later table test.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669b35c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:4619, start at 2024-05-06 19:11:57.18158028 +0800 CST m=+5.962835843 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:57.190 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:57.192 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:57.192 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669b35c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:4619, start at 2024-05-06 19:11:57.18158028 +0800 CST m=+5.962835843 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:57.190 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:57.192 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:57.192 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669b3ec0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:4677, start at 2024-05-06 19:11:57.199902679 +0800 CST m=+5.912991546 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:57.211 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:57.179 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:57.179 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669a9c40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:8335, start at 2024-05-06 19:11:56.560334043 +0800 CST m=+5.246623082 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:56.569 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:56.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:56.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669a9c40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:8335, start at 2024-05-06 19:11:56.560334043 +0800 CST m=+5.246623082 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:56.569 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:56.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:56.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3669ab4c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:8412, start at 2024-05-06 19:11:56.63126392 +0800 CST m=+5.265302956 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:13:56.638 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:11:56.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:01:56.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_reconstruct.usertable not exists for 2-th check, retry later table partition_table.finish_mark exists check diff successfully table test.finish_mark not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 67-th check, retry later wait process cdc.test exit for 1-th time... [Mon May 6 19:12:01 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.60506052.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9812.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:01 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/259c6fbc-4406-4922-b1b4-2e71e68f96e5 {"id":"259c6fbc-4406-4922-b1b4-2e71e68f96e5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993918} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a2e3bcb 259c6fbc-4406-4922-b1b4-2e71e68f96e5 /tidb/cdc/default/default/upstream/7365842730563904823 {"id":7365842730563904823,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/259c6fbc-4406-4922-b1b4-2e71e68f96e5 {"id":"259c6fbc-4406-4922-b1b4-2e71e68f96e5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993918} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a2e3bcb 259c6fbc-4406-4922-b1b4-2e71e68f96e5 /tidb/cdc/default/default/upstream/7365842730563904823 {"id":7365842730563904823,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/259c6fbc-4406-4922-b1b4-2e71e68f96e5 {"id":"259c6fbc-4406-4922-b1b4-2e71e68f96e5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993918} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a2e3bcb 259c6fbc-4406-4922-b1b4-2e71e68f96e5 /tidb/cdc/default/default/upstream/7365842730563904823 {"id":7365842730563904823,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x wait process cdc.test exit for 2-th time... [Mon May 6 19:12:01 CST 2024] <<<<<< START kafka consumer in changefeed_finish case >>>>>> table changefeed_reconstruct.usertable exists check diff successfully wait process 5734 exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:12:02 CST 2024] <<<<<< run test case partition_table success! >>>>>> check diff failed 1-th time, retry later wait process 5734 exit for 2-th time... table test.finish_mark exists check diff successfully table ddl_manager.finish_mark not exists for 68-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: kafka... <<================= /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5734) - No such process wait process 5734 exit for 3-th time... process 5734 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Mon May 6 19:12:03 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.59815983.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... + set +x + tso='449575366670155777 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575366670155777 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:12:03 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.98579859.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:04 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e6e94b3-f96d-463d-8e86-259758f35bf8 {"id":"5e6e94b3-f96d-463d-8e86-259758f35bf8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993921} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a3dbfcf 5e6e94b3-f96d-463d-8e86-259758f35bf8 /tidb/cdc/default/default/upstream/7365842739307424717 {"id":7365842739307424717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e6e94b3-f96d-463d-8e86-259758f35bf8 {"id":"5e6e94b3-f96d-463d-8e86-259758f35bf8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993921} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a3dbfcf 5e6e94b3-f96d-463d-8e86-259758f35bf8 /tidb/cdc/default/default/upstream/7365842739307424717 {"id":7365842739307424717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e6e94b3-f96d-463d-8e86-259758f35bf8 {"id":"5e6e94b3-f96d-463d-8e86-259758f35bf8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993921} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a3dbfcf 5e6e94b3-f96d-463d-8e86-259758f35bf8 /tidb/cdc/default/default/upstream/7365842739307424717 {"id":7365842739307424717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:12:04 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> table ddl_manager.finish_mark not exists for 69-th check, retry later check diff failed 2-th time, retry later [Mon May 6 19:12:04 CST 2024] <<<<<< START kafka consumer in changefeed_pause_resume case >>>>>> check_changefeed_status 127.0.0.1:8300 1f82ecb1-be26-4acd-ab0f-5ead830f7d54 normal + endpoint=127.0.0.1:8300 + changefeed_id=1f82ecb1-be26-4acd-ab0f-5ead830f7d54 + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/1f82ecb1-be26-4acd-ab0f-5ead830f7d54/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1432 0 --:--:-- --:--:-- --:--:-- 1457 + info='{"state":"normal","resolved_ts":449575367444004870,"checkpoint_ts":449575367181860873}' + echo '{"state":"normal","resolved_ts":449575367444004870,"checkpoint_ts":449575367181860873}' {"state":"normal","resolved_ts":449575367444004870,"checkpoint_ts":449575367181860873} ++ echo '{"state":"normal","resolved_ts":449575367444004870,"checkpoint_ts":449575367181860873}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z '' ]] ++ echo '{"state":"normal","resolved_ts":449575367444004870,"checkpoint_ts":449575367181860873}' ++ jq -r .last_error + error_msg=null + [[ ! null == \n\u\l\l ]] ++ echo '{"state":"normal","resolved_ts":449575367444004870,"checkpoint_ts":449575367181860873}' ++ jq -r .last_warning + error_msg=null + [[ ! null == \n\u\l\l ]] + exit 0 run task successfully table kafka_sink_error_resume.t1 exists table kafka_sink_error_resume.t2 exists check diff successfully check diff failed 1-th time, retry later The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {UpstreamID:7365842716379252717 Namespace:default ID:b614f5d7-ee84-4a9e-be1d-09a56753adb3 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-21780?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:11:57.274458642 +0800 CST StartTs:449575365411602437 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0010da480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575365437816838} {CheckpointTs:449575366355320834 MinTableBarrierTs:449575367666040834 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575366355320834, checkpointTs: 449575366355320834, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fa72356-79c0-4914-8d34-9e37857769ef {"id":"5fa72356-79c0-4914-8d34-9e37857769ef","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a206863 5fa72356-79c0-4914-8d34-9e37857769ef /tidb/cdc/default/default/changefeed/info/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {"upstream-id":7365842716379252717,"namespace":"default","changefeed-id":"b614f5d7-ee84-4a9e-be1d-09a56753adb3","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-21780?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:11:57.274458642+08:00","start-ts":449575365411602437,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575365437816838} /tidb/cdc/default/default/changefeed/status/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {"checkpoint-ts":449575366355320834,"min-table-barrier-ts":449575367666040834,"admin-job-type":0} /tidb/cdc/default/default/task/position/5fa72356-79c0-4914-8d34-9e37857769ef/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842716379252717 {"id":7365842716379252717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {UpstreamID:7365842716379252717 Namespace:default ID:b614f5d7-ee84-4a9e-be1d-09a56753adb3 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-21780?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:11:57.274458642 +0800 CST StartTs:449575365411602437 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0010da480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575365437816838} {CheckpointTs:449575366355320834 MinTableBarrierTs:449575367666040834 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575366355320834, checkpointTs: 449575366355320834, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fa72356-79c0-4914-8d34-9e37857769ef {"id":"5fa72356-79c0-4914-8d34-9e37857769ef","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a206863 5fa72356-79c0-4914-8d34-9e37857769ef /tidb/cdc/default/default/changefeed/info/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {"upstream-id":7365842716379252717,"namespace":"default","changefeed-id":"b614f5d7-ee84-4a9e-be1d-09a56753adb3","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-21780?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:11:57.274458642+08:00","start-ts":449575365411602437,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575365437816838} /tidb/cdc/default/default/changefeed/status/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {"checkpoint-ts":449575366355320834,"min-table-barrier-ts":449575367666040834,"admin-job-type":0} /tidb/cdc/default/default/task/position/5fa72356-79c0-4914-8d34-9e37857769ef/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842716379252717 {"id":7365842716379252717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {UpstreamID:7365842716379252717 Namespace:default ID:b614f5d7-ee84-4a9e-be1d-09a56753adb3 SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-21780?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:11:57.274458642 +0800 CST StartTs:449575365411602437 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0010da480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575365437816838} {CheckpointTs:449575366355320834 MinTableBarrierTs:449575367666040834 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575366355320834, checkpointTs: 449575366355320834, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fa72356-79c0-4914-8d34-9e37857769ef {"id":"5fa72356-79c0-4914-8d34-9e37857769ef","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a206863 5fa72356-79c0-4914-8d34-9e37857769ef /tidb/cdc/default/default/changefeed/info/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {"upstream-id":7365842716379252717,"namespace":"default","changefeed-id":"b614f5d7-ee84-4a9e-be1d-09a56753adb3","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-21780?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:11:57.274458642+08:00","start-ts":449575365411602437,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575365437816838} /tidb/cdc/default/default/changefeed/status/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {"checkpoint-ts":449575366355320834,"min-table-barrier-ts":449575367666040834,"admin-job-type":0} /tidb/cdc/default/default/task/position/5fa72356-79c0-4914-8d34-9e37857769ef/b614f5d7-ee84-4a9e-be1d-09a56753adb3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365842716379252717 {"id":7365842716379252717,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:06 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18c055a5-01a0-49be-a87f-d698e33accdb {"id":"18c055a5-01a0-49be-a87f-d698e33accdb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a3e17db 18c055a5-01a0-49be-a87f-d698e33accdb /tidb/cdc/default/default/upstream/7365842737921919484 {"id":7365842737921919484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18c055a5-01a0-49be-a87f-d698e33accdb {"id":"18c055a5-01a0-49be-a87f-d698e33accdb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a3e17db 18c055a5-01a0-49be-a87f-d698e33accdb /tidb/cdc/default/default/upstream/7365842737921919484 {"id":7365842737921919484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18c055a5-01a0-49be-a87f-d698e33accdb {"id":"18c055a5-01a0-49be-a87f-d698e33accdb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993923} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a3e17db 18c055a5-01a0-49be-a87f-d698e33accdb /tidb/cdc/default/default/upstream/7365842737921919484 {"id":7365842737921919484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9914.out cli changefeed create --start-ts=449575366670155777 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-8678?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table ddl_manager.finish_mark not exists for 70-th check, retry later Create changefeed successfully! ID: d4d1760a-5217-4fe5-834a-8a0605c888a6 Info: {"upstream_id":7365842737921919484,"namespace":"default","id":"d4d1760a-5217-4fe5-834a-8a0605c888a6","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-8678?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:12:06.963900557+08:00","start_ts":449575366670155777,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575366670155777,"checkpoint_ts":449575366670155777,"checkpoint_time":"2024-05-06 19:12:01.929"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully table changefeed_pause_resume.t1 not exists for 1-th check, retry later "id": "5fa72356-79c0-4914-8d34-9e37857769ef", "cluster-id": "default" run task successfully check diff successfully wait process cdc.test exit for 1-th time... + set +x [Mon May 6 19:12:08 CST 2024] <<<<<< START kafka consumer in ddl_sequence case >>>>>> table ddl_manager.finish_mark not exists for 71-th check, retry later wait process cdc.test exit for 2-th time... table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:12:09 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> capture_id: 5fa72356-79c0-4914-8d34-9e37857769ef check_processor_table_count http://127.0.0.1:2379 b614f5d7-ee84-4a9e-be1d-09a56753adb3 5fa72356-79c0-4914-8d34-9e37857769ef 1 run task successfully check_processor_table_count http://127.0.0.1:2379 b614f5d7-ee84-4a9e-be1d-09a56753adb3 5fa72356-79c0-4914-8d34-9e37857769ef 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 72-th check, retry later table changefeed_pause_resume.t3 exists check_processor_table_count http://127.0.0.1:2379 b614f5d7-ee84-4a9e-be1d-09a56753adb3 5fa72356-79c0-4914-8d34-9e37857769ef 0 run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 73-th check, retry later ***************** properties ***************** "scanproportion"="0" "updateproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "workload"="core" "insertproportion"="0" "threadcount"="4" "readallfields"="true" "recordcount"="50" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.user"="root" "mysql.db"="changefeed_reconstruct" "readproportion"="0" ********************************************** Run finished, takes 16.650117ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3691.3, Avg(us): 1335, Min(us): 897, Max(us): 3731, 95th(us): 4000, 99th(us): 4000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m table ddl_manager.finish_mark not exists for 74-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) table ddl_sequence.finish_mark not exists for 1-th check, retry later Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } check diff failed 1-th time, retry later table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-b4swd is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-vd7dm-2g9g7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "efb42877afdf5f567e457de46e1acfa603c9be78" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-vd7dm" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-b4swd" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-b4swd" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-b4swd" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-b4swd in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git table ddl_manager.finish_mark not exists for 75-th check, retry later table ddl_sequence.finish_mark not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later check diff successfully Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-174kw is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8fv99-p2w0m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5ca8640c2af62927f0090142bad71ca3241f9335" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8fv99" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-174kw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-174kw" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-174kw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-174kw in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@68712d9d; decorates RemoteLauncher[hudson.remoting.Channel@39aff067:JNLP4-connect connection from 10.233.67.53/10.233.67.53:36118] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 76-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:12:18 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> table ddl_sequence.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@36592b88; decorates RemoteLauncher[hudson.remoting.Channel@34522b02:JNLP4-connect connection from 10.233.72.228/10.233.72.228:48058] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 check diff failed 3-th time, retry later Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 table ddl_sequence.finish_mark not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 77-th check, retry later Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b26480006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1, pid:4546, start at 2024-05-06 19:12:20.887771407 +0800 CST m=+5.966653450 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:20.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:20.882 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:20.882 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] container [Pipeline] { Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN check diff failed 4-th time, retry later [Pipeline] { Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic_avro Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 table ddl_sequence.finish_mark exists check diff successfully table ddl_manager.finish_mark not exists for 78-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b26480006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1, pid:4546, start at 2024-05-06 19:12:20.887771407 +0800 CST m=+5.966653450 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:20.895 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:20.882 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:20.882 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366b27bc0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1, pid:4631, start at 2024-05-06 19:12:20.996726185 +0800 CST m=+6.023575536 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:21.003 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:20.975 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:20.975 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... check diff failed 5-th time, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:12:24 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> Verifying downstream PD is started... table ddl_manager.finish_mark not exists for 79-th check, retry later [Mon May 6 19:12:25 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.59585960.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully table ddl_manager.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiDB... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:12:27 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> wait process cdc.test exit for 2-th time... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:28 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28dee81c-96bd-4136-8d17-2479be426a90 {"id":"28dee81c-96bd-4136-8d17-2479be426a90","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993945} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a9c6ecf 28dee81c-96bd-4136-8d17-2479be426a90 /tidb/cdc/default/default/upstream/7365842849916402284 {"id":7365842849916402284,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28dee81c-96bd-4136-8d17-2479be426a90 {"id":"28dee81c-96bd-4136-8d17-2479be426a90","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993945} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a9c6ecf 28dee81c-96bd-4136-8d17-2479be426a90 /tidb/cdc/default/default/upstream/7365842849916402284 {"id":7365842849916402284,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28dee81c-96bd-4136-8d17-2479be426a90 {"id":"28dee81c-96bd-4136-8d17-2479be426a90","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993945} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9a9c6ecf 28dee81c-96bd-4136-8d17-2479be426a90 /tidb/cdc/default/default/upstream/7365842849916402284 {"id":7365842849916402284,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.6016.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cdc-test-12240?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml Create changefeed successfully! ID: e3717b9d-190e-418e-ae6a-ec1b1a492889 Info: {"upstream_id":7365842849916402284,"namespace":"default","id":"e3717b9d-190e-418e-ae6a-ec1b1a492889","sink_uri":"kafka://127.0.0.1:9092/ticdc-cdc-test-12240?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:12:28.846472929+08:00","start_ts":449575373683556354,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575373683556354,"checkpoint_ts":449575373683556354,"checkpoint_time":"2024-05-06 19:12:28.683"} PASS wait process cdc.test exit for 4-th time... coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 5-th time... Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-wtdb3-nkf0m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7af9281e9e4180f1d8d4fa62da405207f5f16099" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-wtdb3" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 6-th time... process cdc.test already exit [Mon May 6 19:12:29 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> check diff failed 1-th time, retry later + set +x [Mon May 6 19:12:30 CST 2024] <<<<<< START kafka consumer in cdc case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... go: downloading golang.org/x/text v0.14.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 8.08 secs (461490123 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Agent pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_1872-3psjs-2jtwk --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2f5ac7685ced4efea32e64d670b55753d9c50c07" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_1872-3psjs" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" check diff failed 3-th time, retry later Running on pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366beb600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:7943, start at 2024-05-06 19:12:33.537576732 +0800 CST m=+5.767744839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:33.546 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:33.546 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:33.546 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh Verifying downstream PD is started... [Pipeline] { [Pipeline] stage + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] { (Test) [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // container [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366beb600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:7943, start at 2024-05-06 19:12:33.537576732 +0800 CST m=+5.767744839 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:33.546 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:33.546 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:33.546 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366bec880006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:8010, start at 2024-05-06 19:12:33.578253839 +0800 CST m=+5.753536090 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:33.589 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:33.570 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:33.570 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@263eb635; decorates RemoteLauncher[hudson.remoting.Channel@79eddbb8:JNLP4-connect connection from 10.233.67.10/10.233.67.10:54470] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] cache + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=3bfa9c4d-6326-40ec-94e5-c6d1bd3f75a3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-b4swd HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-vd7dm GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-b4swd pingcap_tiflow_pull_cdc_integration_kafka_test_1872-vd7dm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-b4swd GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:12:36 CST 2024] <<<<<< run test case owner_resign success! >>>>>> No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2ee39dd7; decorates RemoteLauncher[hudson.remoting.Channel@750202dc:JNLP4-connect connection from 10.233.90.232/10.233.90.232:53130] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 check diff failed 4-th time, retry later Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/mq_sink_dispatcher Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 5-th time, retry later [Mon May 6 19:12:38 CST 2024] <<<<<< START cdc server in kafka_simple_basic_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.93379339.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Commit message: "fix(br): use failpoint tidb-server instead (#2951)" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:12:39 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:41 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7959988c-99d0-4a6c-90c4-3d2c3e92e4ec {"id":"7959988c-99d0-4a6c-90c4-3d2c3e92e4ec","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9acc56d2 7959988c-99d0-4a6c-90c4-3d2c3e92e4ec /tidb/cdc/default/default/upstream/7365842901413608283 {"id":7365842901413608283,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7959988c-99d0-4a6c-90c4-3d2c3e92e4ec {"id":"7959988c-99d0-4a6c-90c4-3d2c3e92e4ec","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9acc56d2 7959988c-99d0-4a6c-90c4-3d2c3e92e4ec /tidb/cdc/default/default/upstream/7365842901413608283 {"id":7365842901413608283,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7959988c-99d0-4a6c-90c4-3d2c3e92e4ec {"id":"7959988c-99d0-4a6c-90c4-3d2c3e92e4ec","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993958} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9acc56d2 7959988c-99d0-4a6c-90c4-3d2c3e92e4ec /tidb/cdc/default/default/upstream/7365842901413608283 {"id":7365842901413608283,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.cli.9382.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-avro-31818?protocol=simple&encoding-format=avro' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/conf/changefeed.toml Create changefeed successfully! ID: dc94fe80-a987-467b-be57-5e6d40f4f936 Info: {"upstream_id":7365842901413608283,"namespace":"default","id":"dc94fe80-a987-467b-be57-5e6d40f4f936","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-avro-31818?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-06T19:12:41.71125473+08:00","start_ts":449575377068883969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575377068883969,"checkpoint_ts":449575377068883969,"checkpoint_time":"2024-05-06 19:12:41.597"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: kafka... <<================= Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:12:43 CST 2024] <<<<<< run test case sink_hang success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... [2024/05/06 19:12:41.567 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/06 19:12:41.567 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/06 19:12:42.093 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsda81a5be_1997_4061_b1a0_c4098067bf90"] [2024/05/06 19:12:42.104 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs261379b7_1b3f_43f0_b9e9_b92d4cbd83e3"] [2024/05/06 19:12:42.105 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2403b1d8_3286_4f38_b6cc_8006eed67b4b"] [2024/05/06 19:12:42.106 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs08925fb4_d820_4ad4_9a24_1d70ac187537"] [2024/05/06 19:12:42.108 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc1b08cc3_3f71_40e8_8620_4b5a5af3c038"] [2024/05/06 19:12:42.110 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs77607939_21a3_4256_a116_16593553f402"] [2024/05/06 19:12:42.111 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa5e936f0_722a_4d0b_bc35_fbedbe258e99"] [2024/05/06 19:12:42.112 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs2043f883_2234_4652_9d5b_8de132203ce7"] [2024/05/06 19:12:42.160 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 19:12:42.169 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/06 19:12:42.272 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 19:12:42.278 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/06 19:12:42.666 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:42.667 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:42.676 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:42.681 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:42.686 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:42.686 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:42.686 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:42.686 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:42.689 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:42.690 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:42.692 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:42.693 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:42.693 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:42.693 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:42.697 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:42.699 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:42.769 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 19:12:42.770 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/06 19:12:43.075 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 19:12:43.078 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/06 19:12:43.080 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 19:12:43.083 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/06 19:12:43.260 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:43.268 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:43.282 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:43.283 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:43.292 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:43.293 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:43.294 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:43.298 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:43.304 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:43.363 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:43.370 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:43.370 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:43.376 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:43.376 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:43.377 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:43.379 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:43.419 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 19:12:43.422 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/06 19:12:43.790 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:43.800 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:43.869 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:43.870 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 19:12:43.870 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/06 19:12:43.872 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:43.876 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:43.879 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:43.881 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:43.885 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:43.898 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:43.973 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:43.974 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:43.981 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:43.981 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:43.984 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:43.991 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:43.991 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:44.015 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 19:12:44.016 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/06 19:12:44.459 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:44.488 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:44.507 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:44.566 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:44.567 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:44.567 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:44.569 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:44.573 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:44.580 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:44.611 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:44.612 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:44.612 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:44.663 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:44.664 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:44.665 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:44.665 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:44.690 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 19:12:44.695 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/06 19:12:44.761 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 19:12:44.767 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 19:12:44.773 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/06 19:12:44.780 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/06 19:12:45.100 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:45.162 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:45.169 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:45.199 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:45.206 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:45.206 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:45.260 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:45.261 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:45.265 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:45.268 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:45.283 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:45.288 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:45.289 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:45.291 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:45.292 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:45.292 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:45.319 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 19:12:45.321 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/06 19:12:45.589 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 19:12:45.595 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/06 19:12:45.678 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:45.695 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:45.702 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:45.782 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:45.878 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:45.879 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:45.884 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:45.885 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:45.887 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:45.890 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:45.964 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:45.964 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:45.967 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:45.969 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:45.969 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:45.970 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:45.991 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/06 19:12:45.993 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:12:46.301 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:46.361 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:46.372 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] \033[0;36m<<< Run all test success >>>\033[0m [2024/05/06 19:12:46.401 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:46.487 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 19:12:46.490 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:46.493 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 19:12:46.499 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/06 19:12:46.505 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:46.505 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/06 19:12:46.559 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:46.564 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:46.565 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:46.566 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:46.592 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:46.593 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:46.597 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:46.597 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:46.597 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:46.598 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:46.615 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/06 19:12:46.619 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/06 19:12:46.891 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:46.903 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 7.80 secs (477877610 bytes/sec) [Pipeline] { check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:12:46.960 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:46.983 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:47.086 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:47.179 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:47.183 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:47.185 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:47.190 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:47.191 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:47.269 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:47.272 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:47.278 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:47.278 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:47.280 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:47.280 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:47.291 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/06 19:12:47.294 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/06 19:12:47.315 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/06 19:12:47.359 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/06 19:12:47.562 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:47.566 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:47.573 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:47.596 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:12:47.696 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:47.786 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:47.793 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:47.798 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:47.801 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:47.859 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:47.894 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:47.899 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:47.902 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:47.905 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:47.909 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366c9c9c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:7322, start at 2024-05-06 19:12:44.862755139 +0800 CST m=+5.200355516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:44.869 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:44.839 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:44.839 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366c9c9c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:7322, start at 2024-05-06 19:12:44.862755139 +0800 CST m=+5.200355516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:44.869 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:44.839 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:44.839 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366c9ed00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:7392, start at 2024-05-06 19:12:44.989871516 +0800 CST m=+5.269303621 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:44.997 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:44.980 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:44.980 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:12:47.959 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:47.967 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/06 19:12:47.970 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] container [Pipeline] { [2024/05/06 19:12:48.225 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:48.260 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:48.264 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:48.277 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 19:12:48.277 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/06 19:12:48.280 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:48.281 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 19:12:48.282 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/06 19:12:48.375 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Mon May 6 19:12:48 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic_avro case >>>>>> [Pipeline] sh [2024/05/06 19:12:48.471 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:48.473 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:48.489 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:48.489 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:48.498 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:48.577 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:48.588 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:48.598 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:48.605 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:48.611 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:48.664 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:48.664 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/06 19:12:48.676 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [2024/05/06 19:12:48.860 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:48.877 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:12:48.878 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:48.891 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // cache [Pipeline] } [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [2024/05/06 19:12:48.983 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:12:49.088 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:49.090 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:49.159 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:49.160 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/06 19:12:49.160 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:49.164 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/06 19:12:49.166 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:49.196 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:12:49.202 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [2024/05/06 19:12:49.264 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:12:49.271 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:12:49.280 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:12:49.287 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:12:49.290 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 19:12:49.296 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/06 19:12:49.466 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [Pipeline] // dir [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [2024/05/06 19:12:49.492 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:12:49.494 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:12:49.514 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:12:49.590 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:12:49.697 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:12:49.698 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [Pipeline] // timeout [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] // timeout [Pipeline] } [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [2024/05/06 19:12:49.771 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:12:49.776 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:12:49.776 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:12:49.804 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:12:49.870 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:12:49.890 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:12:49.900 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:12:49.905 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:12:49.958 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/06 19:12:49.962 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/06 19:12:49.965 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [Pipeline] // container [Pipeline] sh [2024/05/06 19:12:49.994 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/06 19:12:49.998 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/06 19:12:50.001 +08:00] [INFO] [main.go:178] ["0 insert success: 1000"] [2024/05/06 19:12:50.006 +08:00] [INFO] [main.go:199] ["0 delete success: 500"] [2024/05/06 19:12:50.081 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:12:50.097 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:12:50.104 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:12:50.167 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:12:50.202 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [Pipeline] // stage [Pipeline] } [2024/05/06 19:12:50.372 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:12:50.380 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:12:50.402 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:12:50.460 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:12:50.464 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:12:50.486 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8841.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] // container [Pipeline] } + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=4aeda202-6b09-4fe9-bfeb-a451ae372e90 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-174kw HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8fv99 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-8fv99 pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-174kw GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-174kw GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:12:50 CST 2024] <<<<<< skip test case clustered_index for kafka! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:12:50 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> [Pipeline] { [Pipeline] { [Pipeline] // withEnv [2024/05/06 19:12:50.505 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:12:50.569 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:12:50.585 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:12:50.595 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:12:50.598 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/06 19:12:50.602 +08:00] [INFO] [main.go:178] ["1 insert success: 1400"] [2024/05/06 19:12:50.662 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [Pipeline] } check diff failed 4-th time, retry later [Pipeline] // node [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366cea140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:12340, start at 2024-05-06 19:12:49.830896845 +0800 CST m=+7.344965086 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:49.837 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:49.797 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:49.797 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // podTemplate [Pipeline] } [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [2024/05/06 19:12:50.763 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:12:50.780 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:12:50.792 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:12:50.805 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:12:50.869 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:12:50.903 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/06 19:12:50.960 +08:00] [INFO] [main.go:178] ["0 insert success: 1100"] [2024/05/06 19:12:50.997 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:12:51.006 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] // withEnv [Pipeline] } [Pipeline] cache [2024/05/06 19:12:51.077 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:12:51.083 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:12:51.088 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:12:51.163 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:12:51.181 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:12:51.197 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:12:51.262 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:12:51.271 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:12:51.274 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:12:51.275 +08:00] [INFO] [main.go:178] ["1 insert success: 1500"] [2024/05/06 19:12:51.291 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:12:51.366 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:12:51.391 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:12:51.462 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:12:51.469 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:12:51.496 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:12:51.658 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:12:51.674 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:12:51.758 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:12:51.767 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:12:51.771 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:12:51.785 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] + set +x + tso='449575379360546818 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575379360546818 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/05/06 19:12:51.801 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/06 19:12:51.803 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:12:51.806 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/06 19:12:51.861 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:12:51.863 +08:00] [INFO] [main.go:178] ["0 insert success: 1200"] [2024/05/06 19:12:51.868 +08:00] [INFO] [main.go:199] ["0 delete success: 600"] [2024/05/06 19:12:51.891 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:12:51.891 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:12:51.893 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/06 19:12:51.897 +08:00] [INFO] [main.go:178] ["1 insert success: 1600"] [2024/05/06 19:12:51.960 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:12:52.066 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:12:52.086 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:12:52.089 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:12:52.165 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:12:52.170 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:12:52.358 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:12:52.380 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:12:52.463 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:12:52.475 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:12:52.476 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:12:52.488 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:12:52.499 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366cea140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:12340, start at 2024-05-06 19:12:49.830896845 +0800 CST m=+7.344965086 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:49.837 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:49.797 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:49.797 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366ccad40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:12426, start at 2024-05-06 19:12:47.826590365 +0800 CST m=+5.272578841 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:47.836 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:47.797 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:47.797 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:12:52.577 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/06 19:12:52.580 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:12:52.582 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:12:52.586 +08:00] [INFO] [main.go:178] ["1 insert success: 1700"] [2024/05/06 19:12:52.597 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:12:52.699 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/05/06 19:12:52.699 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:12:52.770 +08:00] [INFO] [main.go:178] ["0 insert success: 1300"] [2024/05/06 19:12:52.770 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:12:52.778 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] ***************** properties ***************** "mysql.port"="4000" "readproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "requestdistribution"="uniform" "insertproportion"="0" "mysql.db"="multi_capture_1" "operationcount"="0" "dotransactions"="false" "threadcount"="2" "readallfields"="true" "scanproportion"="0" "workload"="core" "recordcount"="10" "mysql.user"="root" ********************************************** Run finished, takes 9.805343ms INSERT - Takes(s): 0.0, Count: 9, OPS: 1758.0, Avg(us): 1593, Min(us): 1093, Max(us): 4530, 95th(us): 5000, 99th(us): 5000 [2024/05/06 19:12:52.895 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:12:52.911 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:12:53.006 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:12:53.011 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:12:53.059 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:12:53.081 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:12:53.084 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:12:53.118 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/05/06 19:12:53.160 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:12:53.162 +08:00] [INFO] [main.go:178] ["1 insert success: 1800"] [2024/05/06 19:12:53.163 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:12:53.173 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:12:53.306 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 19:12:53.312 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] check diff failed 5-th time, retry later [2024/05/06 19:12:53.422 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 19:12:53.424 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/05/06 19:12:53.428 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] [2024/05/06 19:12:53.470 +08:00] [INFO] [main.go:178] ["0 insert success: 1400"] [2024/05/06 19:12:53.474 +08:00] [INFO] [main.go:199] ["0 delete success: 700"] [2024/05/06 19:12:53.509 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 19:12:53.561 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/05/06 19:12:53.565 +08:00] [INFO] [main.go:178] ["1 insert success: 1900"] [2024/05/06 19:12:53.571 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 19:12:53.578 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] [2024/05/06 19:12:53.647 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 19:12:53.663 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 19:12:53.734 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 19:12:53.822 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 19:12:53.841 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:12:53.894 +08:00] [INFO] [main.go:178] ["0 insert success: 1500"] [2024/05/06 19:12:53.910 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/05/06 19:12:53.914 +08:00] [INFO] [main.go:178] ["1 insert success: 2000"] [2024/05/06 19:12:53.928 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 19:12:53.933 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 19:12:53.990 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/05/06 19:12:54.013 +08:00] [INFO] [main.go:178] ["73 insert success: 2000"] [2024/05/06 19:12:54.066 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] ***************** properties ***************** "recordcount"="10" "mysql.port"="4000" "requestdistribution"="uniform" "dotransactions"="false" "threadcount"="2" "operationcount"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "readproportion"="0" "mysql.db"="multi_capture_2" "readallfields"="true" "scanproportion"="0" "workload"="core" "mysql.user"="root" "updateproportion"="0" ********************************************** Run finished, takes 9.635059ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1951.3, Avg(us): 1830, Min(us): 1082, Max(us): 4358, 95th(us): 5000, 99th(us): 5000 [2024/05/06 19:12:54.172 +08:00] [INFO] [main.go:178] ["72 insert success: 2000"] [2024/05/06 19:12:54.279 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs57542447_1a06_4b26_b255_524c0d64a88e"] [2024/05/06 19:12:54.284 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] [2024/05/06 19:12:54.292 +08:00] [INFO] [main.go:178] ["1 insert success: 2100"] [2024/05/06 19:12:54.323 +08:00] [INFO] [main.go:178] ["0 insert success: 1600"] [2024/05/06 19:12:54.326 +08:00] [INFO] [main.go:199] ["0 delete success: 800"] [2024/05/06 19:12:54.355 +08:00] [INFO] [main.go:178] ["0 insert success: 1600"] [2024/05/06 19:12:54.359 +08:00] [INFO] [main.go:199] ["0 delete success: 800"] [2024/05/06 19:12:54.362 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLscc2190dc_5610_4652_9137_ae1e71f6fb5a"] [2024/05/06 19:12:54.599 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] \033[0;36m<<< Run all test success >>>\033[0m [2024/05/06 19:12:54.616 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:54.621 +08:00] [INFO] [main.go:178] ["1 insert success: 2200"] [2024/05/06 19:12:54.626 +08:00] [INFO] [main.go:178] ["1 insert success: 2200"] [2024/05/06 19:12:54.694 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:54.709 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:54.744 +08:00] [INFO] [main.go:178] ["0 insert success: 1700"] [2024/05/06 19:12:54.778 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsad2da3f8_d1ec_4513_bc7b_07a29fa617b2"] [2024/05/06 19:12:54.781 +08:00] [INFO] [main.go:178] ["0 insert success: 1700"] [2024/05/06 19:12:54.793 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs0f3f446a_e023_480e_a934_48da3ebdc4d4"] [2024/05/06 19:12:54.794 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsdbab09ca_5c65_42bf_8873_de0dca167b44"] [2024/05/06 19:12:54.798 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs068e3194_87ca_4997_8dd7_170ca79424aa"] [2024/05/06 19:12:54.807 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs68e9537f_68a1_49e7_94df_5ed765f5594e"] [2024/05/06 19:12:54.808 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa5b07b30_e788_4d63_ac54_8f772855106b"] TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:12:52 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "readallfields"="true" "requestdistribution"="uniform" "dotransactions"="false" "scanproportion"="0" "threadcount"="2" "readproportion"="0" "operationcount"="0" "workload"="core" "mysql.user"="root" "mysql.db"="multi_capture_3" "updateproportion"="0" "recordcount"="10" "mysql.port"="4000" "insertproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 10.432207ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1732.4, Avg(us): 2008, Min(us): 1266, Max(us): 4614, 95th(us): 5000, 99th(us): 5000 [2024/05/06 19:12:54.904 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:54.932 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:55.078 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] table test.finish_mark not exists for 1-th check, retry later [Mon May 6 19:12:54 CST 2024] <<<<<< START cdc server in mq_sink_dispatcher case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.1377113773.out server --log-file /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc.log --log-level info --data-dir /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 6-th time, retry later [2024/05/06 19:12:55.159 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:55.264 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:55.279 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:55.282 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:55.283 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:55.291 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:55.297 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:55.298 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:55.299 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:55.299 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:12:55.307 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:55.309 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:55.317 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:12:55.400 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:55.423 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:55.501 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:55.584 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:12:55.697 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:55.764 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:55.766 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:55.771 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:55.776 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:55.785 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:55.785 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:55.787 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:55.787 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:12:55.800 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:55.800 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:55.809 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:12:55.873 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:55.900 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table test.finish_mark not exists for 2-th check, retry later [2024/05/06 19:12:55.960 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:56.009 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:56.105 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:56.258 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:56.263 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:56.276 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:56.282 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:56.282 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:56.282 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:12:56.296 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:56.296 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:56.314 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:56.314 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:56.365 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:12:56.384 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] ***************** properties ***************** "readproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "workload"="core" "mysql.user"="root" "mysql.host"="127.0.0.1" "mysql.port"="4000" "dotransactions"="false" "readallfields"="true" "threadcount"="2" "recordcount"="10" "updateproportion"="0" "scanproportion"="0" "mysql.db"="multi_capture_4" "insertproportion"="0" ********************************************** Run finished, takes 8.833397ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1973.8, Avg(us): 1669, Min(us): 967, Max(us): 3667, 95th(us): 4000, 99th(us): 4000 [Mon May 6 19:12:56 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.89728974.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:12:56.428 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:56.459 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:56.514 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:56.590 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:12:56.694 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:56.704 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:56.714 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:56.761 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:56.765 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:12:56.767 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:56.784 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:56.789 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:56.816 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:56.817 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:56.833 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:12:56.868 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:56.896 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 15.329943394s"] [2024/05/06 19:12:56.907 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:56.907 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:57.006 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:57.078 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:57.188 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:57.202 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:57.264 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:57.269 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:57.270 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:57.274 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:12:57.293 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:57.297 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:57.320 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:57.321 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:57.324 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:12:57.365 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:57.396 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:57.403 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:57.475 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:57.495 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:57.615 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:57.677 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366d47480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:11135, start at 2024-05-06 19:12:55.788735217 +0800 CST m=+6.324542280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:55.795 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:55.762 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:55.762 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366d47480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:11135, start at 2024-05-06 19:12:55.788735217 +0800 CST m=+6.324542280 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:55.795 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:55.762 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:55.762 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366d378c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8k3vn-w9hsl, pid:11219, start at 2024-05-06 19:12:54.767030153 +0800 CST m=+5.253650549 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:54.774 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:54.755 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:54.755 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... check diff successfully [2024/05/06 19:12:57.709 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:57.713 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:57.763 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:12:57.768 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:57.788 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:57.790 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:57.817 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:57.821 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:57.824 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:12:57.828 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:57.886 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:57.899 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:57.924 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 3-th check, retry later [2024/05/06 19:12:57.964 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:58.076 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:58.095 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:58.196 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:58.258 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:58.270 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:12:58.286 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:58.305 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:58.308 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:58.378 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:58.382 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:58.386 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:58.390 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:12:58.412 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:58.464 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:57 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a4f3af9-56dd-4134-bdfc-67bacd1604c3 {"id":"6a4f3af9-56dd-4134-bdfc-67bacd1604c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993975} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b0a10d1 6a4f3af9-56dd-4134-bdfc-67bacd1604c3 /tidb/cdc/default/default/upstream/7365842967309690404 {"id":7365842967309690404,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a4f3af9-56dd-4134-bdfc-67bacd1604c3 {"id":"6a4f3af9-56dd-4134-bdfc-67bacd1604c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993975} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b0a10d1 6a4f3af9-56dd-4134-bdfc-67bacd1604c3 /tidb/cdc/default/default/upstream/7365842967309690404 {"id":7365842967309690404,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6a4f3af9-56dd-4134-bdfc-67bacd1604c3 {"id":"6a4f3af9-56dd-4134-bdfc-67bacd1604c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993975} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b0a10d1 6a4f3af9-56dd-4134-bdfc-67bacd1604c3 /tidb/cdc/default/default/upstream/7365842967309690404 {"id":7365842967309690404,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.13846.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 19:12:58.484 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:58.485 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:58.566 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:58.580 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:58.665 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:58.684 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:58.693 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:12:58.774 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:58.792 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:58.798 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:58.870 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:58.879 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:58.880 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:58.891 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:12:58.916 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:58.978 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:58.994 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:59.000 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:59.076 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:59.094 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:59.158 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:59.173 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:59.185 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:12:59.224 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366d691c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0, pid:7451, start at 2024-05-06 19:12:57.962457912 +0800 CST m=+5.142387010 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:57.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:57.927 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:57.927 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366d691c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0, pid:7451, start at 2024-05-06 19:12:57.962457912 +0800 CST m=+5.142387010 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:57.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:57.927 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:57.927 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366d6a940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-lhjzd-8nwt0, pid:7528, start at 2024-05-06 19:12:58.066132705 +0800 CST m=+5.194866541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:14:58.073 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:12:58.072 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:02:58.072 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:12:59.270 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:59.281 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:59.363 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:59.365 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:59.369 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:59.382 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:12:59.409 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:59.471 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:12:59.486 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:12:59.489 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:12:59 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 {"id":"8b30dccb-aa5c-438f-88b7-6a2e22dd41b2","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb0057 8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 /tidb/cdc/default/default/upstream/7365842945855918447 {"id":7365842945855918447,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 {"id":"8b30dccb-aa5c-438f-88b7-6a2e22dd41b2","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb0057 8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 /tidb/cdc/default/default/upstream/7365842945855918447 {"id":7365842945855918447,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 {"id":"8b30dccb-aa5c-438f-88b7-6a2e22dd41b2","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb0057 8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 /tidb/cdc/default/default/upstream/7365842945855918447 {"id":7365842945855918447,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:12:59 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.90299031.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:12:59.568 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:59.583 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:59.619 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:59.667 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:59.681 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:12:59.706 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:59.715 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:59.770 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.12607.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575381486272518 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575381486272518 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.13885.out cli changefeed create --start-ts=449575381486272518 '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/changefeed.toml [2024/05/06 19:12:59.803 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:12:59.821 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:59.860 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:59.865 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:12:59.886 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:12:59.970 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:12:59.983 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:12:59.993 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:00.067 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:00.078 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:00.163 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:00.180 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:00.193 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:00.269 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:00.270 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] Create changefeed successfully! ID: test Info: {"upstream_id":7365842967309690404,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:13:00.387102371+08:00","start_ts":449575381486272518,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["verify.t"],"partition":"index-value"},{"matcher":["dispatcher.index"],"partition":"index-value","index":"idx_a"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575381486272518,"checkpoint_ts":449575381486272518,"checkpoint_time":"2024-05-06 19:12:58.448"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:13:00.461 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.finish_mark not exists for 4-th check, retry later [2024/05/06 19:13:00.593 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:00.599 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:00.659 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:00.671 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:00.688 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:00.783 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:00.862 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:00.888 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:00.991 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:01.010 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8840.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 19:13:01.094 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:01.103 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:01.113 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:01.164 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:01.164 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:01.201 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:01.287 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:01.287 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:01.295 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:01.299 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:01.322 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:01.381 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:01.411 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:01.431 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:01.494 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:01.502 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:01.515 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] + set +x + tso='449575381922742273 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575381922742273 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:13:01 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1264912651.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s [2024/05/06 19:13:01.581 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:01.585 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:01.622 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:01.707 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:01.711 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:01.712 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:01.725 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:01.769 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:01.810 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:01.833 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:01.874 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] check diff failed 1-th time, retry later + info='{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575381486272518, "checkpoint_time": "2024-05-06 19:12:58.448", "error": null }' + echo '{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575381486272518, "checkpoint_time": "2024-05-06 19:12:58.448", "error": null }' { "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575381486272518, "checkpoint_time": "2024-05-06 19:12:58.448", "error": null } ++ echo '{' '"upstream_id":' 7365842967309690404, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449575381486272518, '"checkpoint_time":' '"2024-05-06' '19:12:58.448",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365842967309690404, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449575381486272518, '"checkpoint_time":' '"2024-05-06' '19:12:58.448",' '"error":' null '}' + message=null + [[ ! null =~ null ]] run task successfully [2024/05/06 19:13:01.958 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:01.966 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:01.984 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:02.017 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:02.063 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:02.100 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:02 GMT < Content-Length: 1271 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a6f4084-bffa-437f-8977-2506d0cc7d10 {"id":"3a6f4084-bffa-437f-8977-2506d0cc7d10","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993979} /tidb/cdc/default/__cdc_meta__/capture/8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 {"id":"8b30dccb-aa5c-438f-88b7-6a2e22dd41b2","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb0057 8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb008b 3a6f4084-bffa-437f-8977-2506d0cc7d10 /tidb/cdc/default/default/upstream/7365842945855918447 {"id":7365842945855918447,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a6f4084-bffa-437f-8977-2506d0cc7d10 {"id":"3a6f4084-bffa-437f-8977-2506d0cc7d10","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993979} /tidb/cdc/default/__cdc_meta__/capture/8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 {"id":"8b30dccb-aa5c-438f-88b7-6a2e22dd41b2","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb0057 8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb008b 3a6f4084-bffa-437f-8977-2506d0cc7d10 /tidb/cdc/default/default/upstream/7365842945855918447 {"id":7365842945855918447,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a6f4084-bffa-437f-8977-2506d0cc7d10 {"id":"3a6f4084-bffa-437f-8977-2506d0cc7d10","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993979} /tidb/cdc/default/__cdc_meta__/capture/8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 {"id":"8b30dccb-aa5c-438f-88b7-6a2e22dd41b2","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb0057 8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb008b 3a6f4084-bffa-437f-8977-2506d0cc7d10 /tidb/cdc/default/default/upstream/7365842945855918447 {"id":7365842945855918447,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s [2024/05/06 19:13:02.186 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:02.195 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:13:02.198 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:02.259 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:02.297 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:02.376 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:02.382 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] table test.finish_mark not exists for 5-th check, retry later [Mon May 6 19:13:02 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.90859087.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + info='{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575381486272518, "checkpoint_time": "2024-05-06 19:12:58.448", "error": null }' + echo '{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575381486272518, "checkpoint_time": "2024-05-06 19:12:58.448", "error": null }' { "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575381486272518, "checkpoint_time": "2024-05-06 19:12:58.448", "error": null } ++ echo '{' '"upstream_id":' 7365842967309690404, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449575381486272518, '"checkpoint_time":' '"2024-05-06' '19:12:58.448",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 1-th time, retry later [2024/05/06 19:13:02.412 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:02.562 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:02.581 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:02.597 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:02.665 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:02.759 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:02.858 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:02.879 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:13:02.888 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:02.890 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:03.004 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:13:03.078 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] + set +x + tso='449575382280568833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575382280568833 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:13:02 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.88818883.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:13:03.177 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:13:03.184 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:13:03.197 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:13:03.211 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:13:03.266 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:13:03.313 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:13:03.325 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:13:03.372 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:13:03.423 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 6-th check, retry later table test.finish_mark not exists for 7-th check, retry later [2024/05/06 19:13:03.469 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs11d0ef0e_d5a6_4bdf_8df4_c8f9373ebe94"] [2024/05/06 19:13:03.497 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:13:03.664 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:13:03.665 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:13:03.686 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] check diff failed 2-th time, retry later [2024/05/06 19:13:03.739 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:13:03.806 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:13:03.872 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:13:03.893 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:13:03.919 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:03.982 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:04.174 +08:00] [INFO] [main.go:178] ["72 insert success: 1800"] [2024/05/06 19:13:04.243 +08:00] [INFO] [main.go:178] ["73 insert success: 1800"] table test.finish_mark not exists for 6-th check, retry later [2024/05/06 19:13:04.308 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:04.334 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:04.466 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs40561fc8_f110_4217_900f_c620ab94ed2f"] [2024/05/06 19:13:04.499 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs7e9751d7_c391_43cc_ade8_f8f34c20df81"] [2024/05/06 19:13:04.522 +08:00] [INFO] [main.go:178] ["72 insert success: 1900"] [2024/05/06 19:13:04.576 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs85694ff9_ca4e_4a69_a534_8389be51f99f"] [2024/05/06 19:13:04.690 +08:00] [INFO] [main.go:178] ["73 insert success: 1900"] [2024/05/06 19:13:04.778 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] table test.finish_mark not exists for 8-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:04 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/06e5f9bf-db69-4b75-ae30-43c4c3b0f025 {"id":"06e5f9bf-db69-4b75-ae30-43c4c3b0f025","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993981} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b215ed2 06e5f9bf-db69-4b75-ae30-43c4c3b0f025 /tidb/cdc/default/default/upstream/7365842998422690262 {"id":7365842998422690262,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/06e5f9bf-db69-4b75-ae30-43c4c3b0f025 {"id":"06e5f9bf-db69-4b75-ae30-43c4c3b0f025","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993981} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b215ed2 06e5f9bf-db69-4b75-ae30-43c4c3b0f025 /tidb/cdc/default/default/upstream/7365842998422690262 {"id":7365842998422690262,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/06e5f9bf-db69-4b75-ae30-43c4c3b0f025 {"id":"06e5f9bf-db69-4b75-ae30-43c4c3b0f025","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993981} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b215ed2 06e5f9bf-db69-4b75-ae30-43c4c3b0f025 /tidb/cdc/default/default/upstream/7365842998422690262 {"id":7365842998422690262,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.12709.out cli changefeed create --start-ts=449575381922742273 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-5067?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [2024/05/06 19:13:04.825 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:04.962 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs90759133_96eb_432c_8e53_2b0023c5cd88"] [2024/05/06 19:13:04.981 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:04.991 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs87fe2a3a_e8f2_4462_8e13_88d25c627be7"] [2024/05/06 19:13:05.069 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4a236a5b_f0ae_45c3_b894_b775ee5ef300"] [2024/05/06 19:13:05.072 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575382482157611, "checkpoint_time": "2024-05-06 19:13:02.247", "error": null }' + echo '{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575382482157611, "checkpoint_time": "2024-05-06 19:13:02.247", "error": null }' { "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575382482157611, "checkpoint_time": "2024-05-06 19:13:02.247", "error": null } ++ echo '{' '"upstream_id":' 7365842967309690404, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449575382482157611, '"checkpoint_time":' '"2024-05-06' '19:13:02.247",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 2-th time, retry later Create changefeed successfully! ID: 2e490184-5216-417c-bfd5-bd991191ca4b Info: {"upstream_id":7365842998422690262,"namespace":"default","id":"2e490184-5216-417c-bfd5-bd991191ca4b","sink_uri":"kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-5067?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:13:05.132967781+08:00","start_ts":449575381922742273,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575381922742273,"checkpoint_ts":449575381922742273,"checkpoint_time":"2024-05-06 19:13:00.113"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:13:05.089 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:05.165 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:05.193 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:05.281 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:05.484 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:05.514 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:05.581 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:05.592 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:05.598 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:05 GMT < Content-Length: 1750 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a6f4084-bffa-437f-8977-2506d0cc7d10 {"id":"3a6f4084-bffa-437f-8977-2506d0cc7d10","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993979} /tidb/cdc/default/__cdc_meta__/capture/4db8ad07-7b97-4c67-8cf8-c9d069703326 {"id":"4db8ad07-7b97-4c67-8cf8-c9d069703326","address":"127.0.0.1:8303","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993982} /tidb/cdc/default/__cdc_meta__/capture/8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 {"id":"8b30dccb-aa5c-438f-88b7-6a2e22dd41b2","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb0057 8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb008b 3a6f4084-bffa-437f-8977-2506d0cc7d10 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb00ad 4db8ad07-7b97-4c67-8cf8-c9d069703326 /tidb/cdc/default/default/upstream/7365842945855918447 {"id":7365842945855918447,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a6f4084-bffa-437f-8977-2506d0cc7d10 {"id":"3a6f4084-bffa-437f-8977-2506d0cc7d10","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993979} /tidb/cdc/default/__cdc_meta__/capture/4db8ad07-7b97-4c67-8cf8-c9d069703326 {"id":"4db8ad07-7b97-4c67-8cf8-c9d069703326","address":"127.0.0.1:8303","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993982} /tidb/cdc/default/__cdc_meta__/capture/8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 {"id":"8b30dccb-aa5c-438f-88b7-6a2e22dd41b2","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb0057 8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb008b 3a6f4084-bffa-437f-8977-2506d0cc7d10 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb00ad 4db8ad07-7b97-4c67-8cf8-c9d069703326 /tidb/cdc/default/default/upstream/7365842945855918447 {"id":7365842945855918447,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a6f4084-bffa-437f-8977-2506d0cc7d10 {"id":"3a6f4084-bffa-437f-8977-2506d0cc7d10","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993979} /tidb/cdc/default/__cdc_meta__/capture/4db8ad07-7b97-4c67-8cf8-c9d069703326 {"id":"4db8ad07-7b97-4c67-8cf8-c9d069703326","address":"127.0.0.1:8303","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993982} /tidb/cdc/default/__cdc_meta__/capture/8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 {"id":"8b30dccb-aa5c-438f-88b7-6a2e22dd41b2","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb0057 8b30dccb-aa5c-438f-88b7-6a2e22dd41b2 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb008b 3a6f4084-bffa-437f-8977-2506d0cc7d10 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9afb00ad 4db8ad07-7b97-4c67-8cf8-c9d069703326 /tidb/cdc/default/default/upstream/7365842945855918447 {"id":7365842945855918447,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.9138.out cli changefeed create --start-ts=449575379360546818 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-capture-test-21427?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8301 [2024/05/06 19:13:05.675 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:05.686 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:05.686 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:05.693 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:05.696 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:05.773 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:05.782 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Create changefeed successfully! ID: d7e4751e-8057-424f-a8cb-8178ab777e4c Info: {"upstream_id":7365842945855918447,"namespace":"default","id":"d7e4751e-8057-424f-a8cb-8178ab777e4c","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-capture-test-21427?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:13:05.790361981+08:00","start_ts":449575379360546818,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575379360546818,"checkpoint_ts":449575379360546818,"checkpoint_time":"2024-05-06 19:12:50.339"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:13:05.875 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:05.980 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd84d0ed4_6cba_4b13_ac18_feaa35ce9268"] [2024/05/06 19:13:05.995 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table test.finish_mark not exists for 7-th check, retry later [2024/05/06 19:13:06.360 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:06 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7df5572b-a754-4755-861e-a3b16c066238 {"id":"7df5572b-a754-4755-861e-a3b16c066238","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993983} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b345ecf 7df5572b-a754-4755-861e-a3b16c066238 /tidb/cdc/default/default/upstream/7365843010755578379 {"id":7365843010755578379,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7df5572b-a754-4755-861e-a3b16c066238 {"id":"7df5572b-a754-4755-861e-a3b16c066238","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993983} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b345ecf 7df5572b-a754-4755-861e-a3b16c066238 /tidb/cdc/default/default/upstream/7365843010755578379 {"id":7365843010755578379,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7df5572b-a754-4755-861e-a3b16c066238 {"id":"7df5572b-a754-4755-861e-a3b16c066238","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714993983} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b345ecf 7df5572b-a754-4755-861e-a3b16c066238 /tidb/cdc/default/default/upstream/7365843010755578379 {"id":7365843010755578379,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8939.out cli changefeed create --start-ts=449575382280568833 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-source-test-30407?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + set +x [Mon May 6 19:13:06 CST 2024] <<<<<< START kafka consumer in resourcecontrol case >>>>>> [2024/05/06 19:13:06.395 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:06.406 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:06.408 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:06.464 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:06.493 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:06.496 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:06.497 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:06.499 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:06.505 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:06.517 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:06.525 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:06.578 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] Create changefeed successfully! ID: fdf2a71c-1abf-4dda-bda6-3cf6317c4bfc Info: {"upstream_id":7365843010755578379,"namespace":"default","id":"fdf2a71c-1abf-4dda-bda6-3cf6317c4bfc","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-source-test-30407?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:13:06.473919706+08:00","start_ts":449575382280568833,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575382280568833,"checkpoint_ts":449575382280568833,"checkpoint_time":"2024-05-06 19:13:01.478"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 9-th check, retry later [2024/05/06 19:13:06.661 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:06.662 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:06.667 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:06.991 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:07.088 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:07.160 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:07.168 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:07.173 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:07.197 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:07.198 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:07.202 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:07.203 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:07.205 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:07.213 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:07.215 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:07.273 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:07.295 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:07.299 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:07.304 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] + set +x [Mon May 6 19:13:07 CST 2024] <<<<<< START kafka consumer in multi_capture case >>>>>> table multi_capture_1.usertable not exists for 1-th check, retry later [2024/05/06 19:13:07.571 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:07.703 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:07.781 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:07.790 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:07.791 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:07.863 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:07.869 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:07.875 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:07.877 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:07.878 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:07.883 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:07.890 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:07.918 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] + set +x [Mon May 6 19:13:07 CST 2024] <<<<<< START kafka consumer in multi_source case >>>>>> go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/coreos/go-semver v0.3.1 table test.finish_mark not exists for 8-th check, retry later [2024/05/06 19:13:07.927 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:07.966 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:07.973 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:08.169 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check diff failed 4-th time, retry later [2024/05/06 19:13:08.390 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] go: downloading github.com/golang/protobuf v1.5.4 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/text v0.14.0 table test.finish_mark not exists for 10-th check, retry later [2024/05/06 19:13:08.483 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:08.488 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:08.489 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:08.662 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:08.680 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:08.682 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:08.686 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:08.691 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:08.693 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:08.759 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:08.763 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:08.765 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:08.777 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:08.780 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:08.824 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:08.977 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:09.073 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:09.074 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:09.091 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:09.199 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table multi_capture_1.usertable not exists for 2-th check, retry later check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449575382482157611, "checkpoint_time": "2024-05-06 19:13:02.247", "error": { "time": "2024-05-06T19:13:06.824075914+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } }' + echo '{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449575382482157611, "checkpoint_time": "2024-05-06 19:13:02.247", "error": { "time": "2024-05-06T19:13:06.824075914+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } }' { "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 449575382482157611, "checkpoint_time": "2024-05-06 19:13:02.247", "error": { "time": "2024-05-06T19:13:06.824075914+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365842967309690404, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 449575382482157611, '"checkpoint_time":' '"2024-05-06' '19:13:02.247",' '"error":' '{' '"time":' '"2024-05-06T19:13:06.824075914+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: index, index: 'idx_a"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365842967309690404, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 449575382482157611, '"checkpoint_time":' '"2024-05-06' '19:13:02.247",' '"error":' '{' '"time":' '"2024-05-06T19:13:06.824075914+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: index, index: 'idx_a"' '}' '}' + message='[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a' + [[ ! [CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a =~ ErrDispatcherFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14135.out cli changefeed update -c test '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/new_changefeed.toml --no-confirm [2024/05/06 19:13:09.271 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:09.278 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:09.292 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:09.292 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:09.293 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:09.457 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] Diff of changefeed config: {Type:update Path:[Config SyncPointInterval] From: To:0xc003984838} {Type:update Path:[Config SyncPointRetention] From: To:0xc003984848} {Type:update Path:[Config Sink DispatchRules 0 Matcher 0] From:verify.t To:dispatcher.index} {Type:delete Path:[Config Sink DispatchRules 1 Matcher 0] From:dispatcher.index To:} {Type:delete Path:[Config Sink DispatchRules 1 PartitionRule] From:index-value To:} {Type:delete Path:[Config Sink DispatchRules 1 IndexName] From:idx_a To:} {Type:update Path:[Config Consistent] From: To:0xc0013a8150} Update changefeed config successfully! ID: test Info: {"upstream_id":7365842967309690404,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:13:00.387102371+08:00","start_ts":449575381486272518,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","dispatchers":[{"matcher":["dispatcher.index"],"partition":"index-value"}],"encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"failed","error":{"addr":"127.0.0.1:8300","code":"CDC:ErrDispatcherFailed","message":"[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a"},"creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":0,"checkpoint_ts":449575382482157611,"checkpoint_time":"2024-05-06 19:13:02.247"} PASS [2024/05/06 19:13:09.463 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:09.466 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:09.470 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:09.487 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:09.501 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:09.692 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] table resourcecontrol.finish_mark not exists for 1-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:13:09.788 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:09.790 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:09.883 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] check diff failed 5-th time, retry later [2024/05/06 19:13:09.987 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:10.093 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:10.165 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:10.172 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:10.176 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:10.182 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:10.264 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:10.269 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:10.275 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:10.282 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:10.294 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:10.300 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:10.394 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:10.411 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:10.464 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:10.484 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/google/btree v1.1.2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/kr/pretty v0.3.1 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d table test.finish_mark not exists for 9-th check, retry later [2024/05/06 19:13:10.584 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:10.698 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron v1.2.0 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/apache/thrift v0.16.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 [2024/05/06 19:13:10.777 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:10.795 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:10.799 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:10.804 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:10.961 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:10.967 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:10.978 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:10.978 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] table test.finish_mark not exists for 11-th check, retry later table resourcecontrol.finish_mark exists go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14172.out cli changefeed resume -c test [2024/05/06 19:13:11.061 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:11.068 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:11.185 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:11.196 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:11.197 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] check diff successfully start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [2024/05/06 19:13:11.296 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:11.383 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:11.493 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:11.561 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:11.580 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:11.587 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:11.589 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:11.663 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:11.669 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:11.672 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:11.683 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:11.705 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table multi_capture_1.usertable exists table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... check diff successfully [2024/05/06 19:13:11.889 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:11.959 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:11.987 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:12.097 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:12.190 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:12.262 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:12.283 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:12.285 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:12.287 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:12.298 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:12.303 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:12.308 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:12.314 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:12.321 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.finish_mark not exists for 10-th check, retry later wait process cdc.test exit for 2-th time... [2024/05/06 19:13:12.383 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:12.401 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:12.428 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:12.595 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:13:12 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> [2024/05/06 19:13:12.765 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] table test.finish_mark not exists for 12-th check, retry later [2024/05/06 19:13:12.889 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:12.909 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:12.910 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:12.915 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:12.964 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:12.965 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:12.969 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:12.982 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:12.986 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:13.023 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:13.025 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:13.025 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:13.103 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:13.178 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:13.377 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:13.476 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:13.483 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:13.507 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:13.509 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:13.518 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:13.565 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:13.566 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:13.591 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:13.592 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:13.597 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsa4d0b0d9_9af5_4773_980a_dd0088b70259"] + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575382482157611, "checkpoint_time": "2024-05-06 19:13:02.247", "error": null }' + echo '{ "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575382482157611, "checkpoint_time": "2024-05-06 19:13:02.247", "error": null }' { "upstream_id": 7365842967309690404, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 449575382482157611, "checkpoint_time": "2024-05-06 19:13:02.247", "error": null } ++ echo '{' '"upstream_id":' 7365842967309690404, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449575382482157611, '"checkpoint_time":' '"2024-05-06' '19:13:02.247",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365842967309690404, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 449575382482157611, '"checkpoint_time":' '"2024-05-06' '19:13:02.247",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table test.finish_mark not exists for 1-th check, retry later table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later [2024/05/06 19:13:13.638 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:13.669 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:13:13.988 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:14.078 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:14.078 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:14.079 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:14.160 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:14.169 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:14.173 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:14.174 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:14.174 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:14.183 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:14.202 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:14.233 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] table test.finish_mark not exists for 11-th check, retry later go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 [2024/05/06 19:13:14.484 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:14.667 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:14.671 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:14.673 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:14.860 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:14.861 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:14.866 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:14.873 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:14.881 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:14.889 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:14.904 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] table test.finish_mark not exists for 13-th check, retry later [2024/05/06 19:13:15.019 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:15.103 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:15.103 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:15.158 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs33f7dacd_4ef2_4260_811b_bead9eeb6c0d"] [2024/05/06 19:13:15.164 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs76b05f8f_d61a_4c7b_a65b_4e469031ad8e"] [2024/05/06 19:13:15.262 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:15.264 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:15.287 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:15.290 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd table multi_capture_4.usertable exists check diff failed 1-th time, retry later [2024/05/06 19:13:15.567 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:15.579 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:13:15.616 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs757f5a07_72a8_4524_a032_947816a39222"] [2024/05/06 19:13:15.663 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:15.664 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:15.672 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:15.672 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore [2024/05/06 19:13:15.729 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:15.733 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:15.758 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:15.761 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:13:15.791 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6e5cf221_54c3_4127_888c_8908cc843bad"] Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later [2024/05/06 19:13:16.079 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:13:16.098 +08:00] [INFO] [main.go:178] ["73 insert success: 1700"] [2024/05/06 19:13:16.171 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsc93c94bf_8471_4864_9552_419a249ca9f6"] [2024/05/06 19:13:16.192 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] table test.finish_mark not exists for 12-th check, retry later [2024/05/06 19:13:16.277 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:16.279 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:16.280 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:16.284 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:16.284 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:16.329 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:16.330 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:16.360 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:16.368 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:16.508 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:16.584 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] table test.finish_mark not exists for 14-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... [2024/05/06 19:13:16.858 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:16.872 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:16.876 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:16.876 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:16.881 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:16.884 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:16.932 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:16.933 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:16.977 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 19.16 secs (194578227 bytes/sec) [Pipeline] { [Pipeline] cache [2024/05/06 19:13:16.988 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:17.066 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:17.082 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:17.462 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:17.472 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:17.475 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:17.482 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:17.488 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] check diff successfully wait process cdc.test exit for 2-th time... [2024/05/06 19:13:17.498 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:17.593 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:17.609 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:17.629 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:17.658 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:17.699 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:17.707 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] ***************** properties ***************** "requestdistribution"="uniform" "updateproportion"="0" "mysql.port"="4000" "recordcount"="20" "insertproportion"="0" "workload"="core" "threadcount"="2" "mysql.user"="root" "readproportion"="0" "mysql.db"="multi_capture_1" "scanproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "readallfields"="true" ********************************************** Run finished, takes 25.390176ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1221.5, Avg(us): 2479, Min(us): 896, Max(us): 9045, 95th(us): 9000, 99th(us): 10000 ***************** properties ***************** "readproportion"="0" "mysql.user"="root" "dotransactions"="false" "requestdistribution"="uniform" "operationcount"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "scanproportion"="0" "threadcount"="2" "recordcount"="20" "mysql.db"="multi_capture_2" "workload"="core" "updateproportion"="0" "mysql.port"="4000" "insertproportion"="0" ********************************************** Run finished, takes 11.887573ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1907.9, Avg(us): 1108, Min(us): 580, Max(us): 1999, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "threadcount"="2" "operationcount"="0" "readproportion"="0" "insertproportion"="0" "dotransactions"="false" "workload"="core" "scanproportion"="0" "recordcount"="20" "mysql.user"="root" "readallfields"="true" "updateproportion"="0" "requestdistribution"="uniform" "mysql.db"="multi_capture_3" "mysql.host"="127.0.0.1" "mysql.port"="4000" ********************************************** Run finished, takes 10.79154ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2124.5, Avg(us): 1034, Min(us): 564, Max(us): 1746, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "requestdistribution"="uniform" "mysql.user"="root" "mysql.db"="multi_capture_4" "threadcount"="2" "workload"="core" "mysql.host"="127.0.0.1" "dotransactions"="false" "insertproportion"="0" "scanproportion"="0" "readproportion"="0" "recordcount"="20" "operationcount"="0" "readallfields"="true" "mysql.port"="4000" "updateproportion"="0" ********************************************** Run finished, takes 11.349672ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1971.5, Avg(us): 1085, Min(us): 554, Max(us): 1671, 95th(us): 2000, 99th(us): 2000 [2024/05/06 19:13:17.899 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:17.967 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:17.968 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:17.977 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:17.981 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:17.988 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:13:17 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> table test.finish_mark not exists for 13-th check, retry later [2024/05/06 19:13:18.109 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:18.163 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:18.190 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:18.196 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:18.201 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs83728457_d34b_496f_a803_6a1d3c537b70"] [2024/05/06 19:13:18.266 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:18.275 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd6014859_bdf2_42fb_b776_365185638ac3"] [2024/05/06 19:13:18.286 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:18.466 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:18.498 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] check diff failed 2-th time, retry later [2024/05/06 19:13:18.578 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:18.586 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:18.586 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:18.595 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] table test.finish_mark not exists for 15-th check, retry later [2024/05/06 19:13:18.789 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:18.834 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:18.968 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:18.968 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:18.986 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:19.012 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:19.061 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:19.072 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:19.082 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:19.084 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:19.188 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:19.265 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:19.374 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:19.379 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:19.380 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:19.467 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:19.490 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:19.680 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:19.704 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:19.771 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:19.785 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:19.795 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:19.799 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] check diff successfully [2024/05/06 19:13:19.868 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:19.873 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:19.886 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:19.980 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:19.987 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table test.finish_mark not exists for 14-th check, retry later [2024/05/06 19:13:11.446 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 19:13:11.446 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365842539599567596] [2024/05/06 19:13:11.446 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:13:11.446 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 19:13:11.446 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 19:13:11.446 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:13:11.448 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/06 19:13:11.448 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365842539599567596] [2024/05/06 19:13:11.448 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:13:11.448 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/06 19:13:11.448 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/06 19:13:11.448 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/06 19:13:11.449 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/06 19:13:11.449 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/06 19:13:11.451 +08:00] [INFO] [store_cache.go:477] ["change store resolve state"] [store=1] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] [2024/05/06 19:13:20.096 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:20.163 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:20.190 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:20.263 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:20.274 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later [2024/05/06 19:13:20.385 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:20.468 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:20.484 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:20.491 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:20.500 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:20.560 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:20.579 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:20.589 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] table test.finish_mark not exists for 16-th check, retry later [2024/05/06 19:13:20.596 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:20.664 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:20.670 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:20.762 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:20.768 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] wait process cdc.test exit for 2-th time... [2024/05/06 19:13:20.861 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:20.886 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:20.907 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:21.004 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:21.078 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:21.100 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:21.168 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:21.177 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:21.185 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:21.201 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:21.213 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:21.264 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:21.269 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:21.270 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] wait process cdc.test exit for 3-th time... [2024/05/06 19:13:21.377 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:21.388 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:21.570 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:21.583 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:21.602 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:21.453 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/06 19:13:21.453 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/05/06 19:13:21.453 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/06 19:13:21.453 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/05/06 19:13:21.453 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/06 19:13:21.453 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/06 19:13:21.453 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/06 19:13:21.453 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 19:13:21.453 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/05/06 19:13:21.453 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/05/06 19:13:21.453 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/06 19:13:21.453 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/05/06 19:13:21.453 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] [2024/05/06 19:13:21.697 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:21.758 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:21.778 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:21.861 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:21.869 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:21.869 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:21 CST 2024] <<<<<< run test case multi_capture success! >>>>>> [2024/05/06 19:13:21.879 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:21.884 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:21.893 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:21.902 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:21.908 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:21.963 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:21.995 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:22.178 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:22.193 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:22.280 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:22.359 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:22.384 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366ed1900016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:7119, start at 2024-05-06 19:13:21.015704299 +0800 CST m=+5.123327836 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:21.022 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:20.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:20.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366ed1900016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:7119, start at 2024-05-06 19:13:21.015704299 +0800 CST m=+5.123327836 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:21.022 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:20.996 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:20.996 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d366ed4a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:7203, start at 2024-05-06 19:13:21.215457074 +0800 CST m=+5.272993804 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:21.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:21.194 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:21.194 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later table test.finish_mark not exists for 15-th check, retry later [2024/05/06 19:13:22.405 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:22.504 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:22.509 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:22.562 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] [2024/05/06 19:13:22.565 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:22.568 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:22.573 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:22.577 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:22.578 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:22.589 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:22.615 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:22.772 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:22.890 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:22.991 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:23.067 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] table test.finish_mark exists check diff successfully [2024/05/06 19:13:23.175 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:23.206 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:23.208 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:23.212 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:23.214 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:23.262 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:23.266 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:23.271 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:23.279 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:23.291 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] wait process cdc.test exit for 1-th time... [2024/05/06 19:13:23.464 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:23.557 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:23.687 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:23.765 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:23.776 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:23.815 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:23.821 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:23.859 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:23.866 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:23.868 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:23.878 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:23.894 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] wait process cdc.test exit for 2-th time... [2024/05/06 19:13:23.946 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:23.982 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:24.094 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:24.175 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:24.196 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [Mon May 6 19:13:24 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.86018603.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:13:24.272 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:24.316 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:24.320 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:24.321 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:24.326 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:24.407 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:24.428 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] table test.finish_mark not exists for 16-th check, retry later wait process cdc.test exit for 3-th time... [2024/05/06 19:13:24.505 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:24.573 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:24.606 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:24.695 +08:00] [INFO] [main.go:178] ["72 insert success: 1000"] cdc.test: no process found wait process cdc.test exit for 4-th time... check diff failed 5-th time, retry later [2024/05/06 19:13:24.777 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd5a9904f_1f57_430b_8d52_9bc57b06f2ce"] [2024/05/06 19:13:24.800 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:24.801 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/06 19:13:24.891 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:24.898 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:24.943 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:24.963 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] process cdc.test already exit [Mon May 6 19:13:24 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/05/06 19:13:25.031 +08:00] [INFO] [main.go:178] ["72 insert success: 1100"] [2024/05/06 19:13:25.105 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:25.188 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:25.262 +08:00] [INFO] [main.go:178] ["73 insert success: 1100"] [2024/05/06 19:13:25.374 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:25.412 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:25.453 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:25.515 +08:00] [INFO] [main.go:178] ["72 insert success: 1200"] [2024/05/06 19:13:25.523 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:25.576 +08:00] [INFO] [main.go:178] ["73 insert success: 1200"] [2024/05/06 19:13:25.685 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:25.721 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:25.764 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:25.790 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3adaf71f_bf84_485b_892a_af362d4b904c"] [2024/05/06 19:13:25.799 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs24f6757c_da73_4f47_ab6d_680108c123ab"] [2024/05/06 19:13:25.819 +08:00] [INFO] [main.go:178] ["72 insert success: 1300"] [2024/05/06 19:13:25.831 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:25.892 +08:00] [INFO] [main.go:178] ["73 insert success: 1300"] [2024/05/06 19:13:26.066 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:26.105 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:26.218 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:26.228 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] table test.finish_mark exists check diff successfully [2024/05/06 19:13:26.258 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:26.268 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:26.276 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:26.279 +08:00] [INFO] [main.go:178] ["72 insert success: 1400"] [2024/05/06 19:13:26.305 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:26.381 +08:00] [INFO] [main.go:178] ["73 insert success: 1400"] [2024/05/06 19:13:26.503 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:26.577 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:26.675 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:26.678 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] [2024/05/06 19:13:26.680 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:26.691 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:26.697 +08:00] [INFO] [main.go:178] ["72 insert success: 1500"] [2024/05/06 19:13:26.699 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:26.732 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:26.765 +08:00] [INFO] [main.go:178] ["73 insert success: 1500"] check diff failed 6-th time, retry later [2024/05/06 19:13:26.833 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:26.986 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:26.990 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:27.004 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:27.010 +08:00] [INFO] [main.go:178] ["72 insert success: 1600"] [2024/05/06 19:13:27.015 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:27.044 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:27.085 +08:00] [INFO] [main.go:178] ["73 insert success: 1600"] [2024/05/06 19:13:27.152 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:27 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4666a2a6-f6f9-43ee-a668-97c49518b4bc {"id":"4666a2a6-f6f9-43ee-a668-97c49518b4bc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994004} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b8c87c5 4666a2a6-f6f9-43ee-a668-97c49518b4bc /tidb/cdc/default/default/upstream/7365843109605650266 {"id":7365843109605650266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4666a2a6-f6f9-43ee-a668-97c49518b4bc {"id":"4666a2a6-f6f9-43ee-a668-97c49518b4bc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994004} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b8c87c5 4666a2a6-f6f9-43ee-a668-97c49518b4bc /tidb/cdc/default/default/upstream/7365843109605650266 {"id":7365843109605650266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4666a2a6-f6f9-43ee-a668-97c49518b4bc {"id":"4666a2a6-f6f9-43ee-a668-97c49518b4bc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994004} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b8c87c5 4666a2a6-f6f9-43ee-a668-97c49518b4bc /tidb/cdc/default/default/upstream/7365843109605650266 {"id":7365843109605650266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.8659.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-only-block-related-table [2024/05/06 19:13:27.325 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs238d401c_5d55_427f_8068_e570375c9901"] [2024/05/06 19:13:27.358 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:27.368 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:27.383 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:27.391 +08:00] [INFO] [main.go:178] ["72 insert success: 1700"] [2024/05/06 19:13:27.392 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:27.424 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:27.488 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7365843109605650266,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:13:27.750069952+08:00","start_ts":449575389127507973,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575389127507973,"checkpoint_ts":449575389127507973,"checkpoint_time":"2024-05-06 19:13:27.597"} PASS [2024/05/06 19:13:27.670 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:27.696 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:27.716 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:27.723 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:27.764 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:27.775 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:13:27.807 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:27.859 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:27.998 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:28.008 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsef5c0d5c_def8_4bca_b0df_8c3a8046ee40"] [2024/05/06 19:13:28.014 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:28.043 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] table test.t2 not exists for 1-th check, retry later [2024/05/06 19:13:28.059 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:28.095 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:28.109 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:28.138 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:28.193 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:28.309 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:28.320 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/06 19:13:28.331 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/06 19:13:28.335 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:28.371 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:28.389 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:28.434 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:28.446 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] check diff successfully [2024/05/06 19:13:28.621 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:28.629 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/06 19:13:28.640 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/06 19:13:28.645 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:28.682 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:28.701 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:28.746 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:28.759 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:28.935 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:28.938 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/06 19:13:28.952 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/06 19:13:28.961 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:29.024 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:29.077 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/06 19:13:29.279 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:29.281 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/06 19:13:29.300 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/06 19:13:29.310 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + set +x [Mon May 6 19:13:29 CST 2024] <<<<<< START kafka consumer in ddl_only_block_related_table case >>>>>> table test.t2 not exists for 2-th check, retry later [2024/05/06 19:13:29.575 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:29.578 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/06 19:13:29.612 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/06 19:13:29.619 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:29.867 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:29.871 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/06 19:13:29.921 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/06 19:13:29.929 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/06 19:13:30.189 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/06 19:13:30.194 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/06 19:13:30.238 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/06 19:13:30.246 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] table test.finish_mark not exists for 1-th check, retry later [2024/05/06 19:13:30.485 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/06 19:13:30.545 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] \033[0;36m<<< Run all test success >>>\033[0m [2024/05/06 19:13:31.161 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v49.594118011s"] [2024/05/06 19:13:31.673 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 50.106703781s"] table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later table mark.finish_mark_1 not exists for 1-th check, retry later table test.t2 not exists for 3-th check, retry later check diff failed 1-th time, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh: line 1: 14278 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/new_changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 2-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc) 3727343616 bytes in 14.56 secs (256075855 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { table test.t2 not exists for 4-th check, retry later [Pipeline] sh start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... [Pipeline] sh table mark.finish_mark_1 not exists for 2-th check, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] // stage + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 table test.finish_mark not exists for 3-th check, retry later check diff failed 2-th time, retry later table test.t2 not exists for 5-th check, retry later table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later table mark.finish_mark_1 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 4-th check, retry later check diff failed 3-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table test.t2 not exists for 6-th check, retry later table ddl_only_block_related_table.finish_mark exists table mark.finish_mark_1 not exists for 4-th check, retry later wait process 8606 exit for 1-th time... table test.finish_mark not exists for 5-th check, retry later wait process 8606 exit for 2-th time... check diff failed 4-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.t2 not exists for 7-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8606) - No such process wait process 8606 exit for 3-th time... process 8606 already exit [Mon May 6 19:13:39 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.87848786.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_1 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table test.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] } check diff failed 5-th time, retry later table mark.finish_mark_1 not exists for 6-th check, retry later table test.t2 not exists for 8-th check, retry later Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:42 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843109605650266 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:13:27.750069952 +0800 CST StartTs:449575389127507973 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001403ef0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575389166567426} {CheckpointTs:449575392889012227 MinTableBarrierTs:449575392889012227 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e342e53c-c086-411c-8cd6-bf885a0ace06 {"id":"e342e53c-c086-411c-8cd6-bf885a0ace06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994019} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b8c889e e342e53c-c086-411c-8cd6-bf885a0ace06 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843109605650266,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:13:27.750069952+08:00","start-ts":449575389127507973,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575389166567426} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575392889012227,"min-table-barrier-ts":449575392889012227,"admin-job-type":0} /tidb/cdc/default/default/task/position/e342e53c-c086-411c-8cd6-bf885a0ace06/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843109605650266 {"id":7365843109605650266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843109605650266 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:13:27.750069952 +0800 CST StartTs:449575389127507973 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001403ef0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575389166567426} {CheckpointTs:449575392889012227 MinTableBarrierTs:449575392889012227 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e342e53c-c086-411c-8cd6-bf885a0ace06 {"id":"e342e53c-c086-411c-8cd6-bf885a0ace06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994019} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b8c889e e342e53c-c086-411c-8cd6-bf885a0ace06 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843109605650266,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:13:27.750069952+08:00","start-ts":449575389127507973,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575389166567426} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575392889012227,"min-table-barrier-ts":449575392889012227,"admin-job-type":0} /tidb/cdc/default/default/task/position/e342e53c-c086-411c-8cd6-bf885a0ace06/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843109605650266 {"id":7365843109605650266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843109605650266 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:13:27.750069952 +0800 CST StartTs:449575389127507973 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001403ef0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575389166567426} {CheckpointTs:449575392889012227 MinTableBarrierTs:449575392889012227 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575392889012227, checkpointTs: 449575392889012227, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e342e53c-c086-411c-8cd6-bf885a0ace06 {"id":"e342e53c-c086-411c-8cd6-bf885a0ace06","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994019} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b8c889e e342e53c-c086-411c-8cd6-bf885a0ace06 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843109605650266,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:13:27.750069952+08:00","start-ts":449575389127507973,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575389166567426} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575392889012227,"min-table-barrier-ts":449575392889012227,"admin-job-type":0} /tidb/cdc/default/default/task/position/e342e53c-c086-411c-8cd6-bf885a0ace06/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843109605650266 {"id":7365843109605650266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_ts_not_forward ddl-only-block-related-table table test.finish_mark not exists for 7-th check, retry later check diff failed 6-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 455a7459-08bc-4b4c-9c2e-96a4e5f66e97 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=455a7459-08bc-4b4c-9c2e-96a4e5f66e97 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 455a7459-08bc-4b4c-9c2e-96a4e5f66e97 -s + info='{ "upstream_id": 7365842730563904823, "namespace": "default", "id": "455a7459-08bc-4b4c-9c2e-96a4e5f66e97", "state": "finished", "checkpoint_tso": 449575390142791686, "checkpoint_time": "2024-05-06 19:13:31.470", "error": null }' + echo '{ "upstream_id": 7365842730563904823, "namespace": "default", "id": "455a7459-08bc-4b4c-9c2e-96a4e5f66e97", "state": "finished", "checkpoint_tso": 449575390142791686, "checkpoint_time": "2024-05-06 19:13:31.470", "error": null }' { "upstream_id": 7365842730563904823, "namespace": "default", "id": "455a7459-08bc-4b4c-9c2e-96a4e5f66e97", "state": "finished", "checkpoint_tso": 449575390142791686, "checkpoint_time": "2024-05-06 19:13:31.470", "error": null } ++ echo '{' '"upstream_id":' 7365842730563904823, '"namespace":' '"default",' '"id":' '"455a7459-08bc-4b4c-9c2e-96a4e5f66e97",' '"state":' '"finished",' '"checkpoint_tso":' 449575390142791686, '"checkpoint_time":' '"2024-05-06' '19:13:31.470",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7365842730563904823, '"namespace":' '"default",' '"id":' '"455a7459-08bc-4b4c-9c2e-96a4e5f66e97",' '"state":' '"finished",' '"checkpoint_tso":' 449575390142791686, '"checkpoint_time":' '"2024-05-06' '19:13:31.470",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:13:38 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> table test.t2 not exists for 9-th check, retry later table mark.finish_mark_1 not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367036cc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:15454, start at 2024-05-06 19:13:43.892090328 +0800 CST m=+5.144809398 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:43.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:43.859 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:43.859 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367036cc0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:15454, start at 2024-05-06 19:13:43.892090328 +0800 CST m=+5.144809398 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:43.900 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:43.859 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:43.859 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367038500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:15525, start at 2024-05-06 19:13:43.997914763 +0800 CST m=+5.199970246 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:44.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:44.006 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:44.006 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table test.finish_mark not exists for 8-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) table test.t2 not exists for 10-th check, retry later [Pipeline] } [Pipeline] } table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark exists check diff successfully [Pipeline] // cache wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36705e240007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:10519, start at 2024-05-06 19:13:46.385377153 +0800 CST m=+5.316859378 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:46.392 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:46.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:46.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Mon May 6 19:13:47 CST 2024] <<<<<< START cdc server in kafka_column_selector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.1695816960.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // cache [Pipeline] } table test.t2 not exists for 11-th check, retry later [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // timeout [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // container [Pipeline] sh wait process cdc.test exit for 3-th time... table mark.finish_mark_2 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=0df58b6f-ea89-48d2-abdb-049377d618b0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-wtdb3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7 pingcap_tiflow_pull_cdc_integration_kafka_test_1872-wtdb3 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:48 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> [Pipeline] // withCredentials [Pipeline] } [Pipeline] // withCredentials [Pipeline] } + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G13 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36705e240007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:10519, start at 2024-05-06 19:13:46.385377153 +0800 CST m=+5.316859378 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:46.392 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:46.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:46.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36705ee40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:10600, start at 2024-05-06 19:13:46.466818815 +0800 CST m=+5.310388897 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:46.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:46.474 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:46.474 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // timeout [Pipeline] } Run cases: tiflash region_merge common_1 PROW_JOB_ID=2d344514-6201-4580-9cf8-032f1375a90c JENKINS_NODE_COOKIE=04106be7-a314-4a87-897d-be84285bfd1e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1787435357046837249","prowjobid":"2d344514-6201-4580-9cf8-032f1375a90c","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"c710066a51e20310bd1ae34af39296cecf567182","base_link":"https://github.com/pingcap/tiflow/commit/c710066a51e20310bd1ae34af39296cecf567182","pulls":[{"number":11024,"author":"sdojjy","sha":"880c4938eb45789c9feb7606b94fbf04e0a60b7a","title":"kafka(ticdc): Fix java example license issue","link":"https://github.com/pingcap/tiflow/pull/11024","commit_link":"https://github.com/pingcap/tiflow/pull/11024/commits/880c4938eb45789c9feb7606b94fbf04e0a60b7a","author_link":"https://github.com/sdojjy"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1872 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1787435357046837249 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/1872/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-3psjs GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_1872-3psjs pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=1872 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 1-th time, retry later [Pipeline] // timeout run task failed 1-th time, retry later [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:50 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25d73a6d-bdaa-4d3a-8da2-d29c6804e932 {"id":"25d73a6d-bdaa-4d3a-8da2-d29c6804e932","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994027} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9be7cbc7 25d73a6d-bdaa-4d3a-8da2-d29c6804e932 /tidb/cdc/default/default/upstream/7365843207962446679 {"id":7365843207962446679,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25d73a6d-bdaa-4d3a-8da2-d29c6804e932 {"id":"25d73a6d-bdaa-4d3a-8da2-d29c6804e932","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994027} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9be7cbc7 25d73a6d-bdaa-4d3a-8da2-d29c6804e932 /tidb/cdc/default/default/upstream/7365843207962446679 {"id":7365843207962446679,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25d73a6d-bdaa-4d3a-8da2-d29c6804e932 {"id":"25d73a6d-bdaa-4d3a-8da2-d29c6804e932","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994027} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9be7cbc7 25d73a6d-bdaa-4d3a-8da2-d29c6804e932 /tidb/cdc/default/default/upstream/7365843207962446679 {"id":7365843207962446679,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.cli.17009.out cli changefeed create --start-ts=449575394242461697 '--sink-uri=kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json&partition-num=1&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/conf/changefeed.toml ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] } check_ts_not_forward ddl-only-block-related-table [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // podTemplate [Pipeline] } Create changefeed successfully! ID: test Info: {"upstream_id":7365843207962446679,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json\u0026partition-num=1\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:13:50.684119406+08:00","start_ts":449575394242461697,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"column_selectors":[{"matcher":["test.t1"],"columns":["a","b"]},{"matcher":["test.*"],"columns":["*","!b"]},{"matcher":["test1.t1"],"columns":["column*","!column1"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575394242461697,"checkpoint_ts":449575394242461697,"checkpoint_time":"2024-05-06 19:13:47.109"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] } table test.t2 exists check diff successfully wait process cdc.test exit for 1-th time... + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11969.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 2-th check, retry later + set +x wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670b2240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5, pid:21600, start at 2024-05-06 19:13:51.757713076 +0800 CST m=+5.503838965 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:51.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:51.753 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:51.753 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... Starting build checksum checker... go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/spf13/cobra v1.8.0 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading golang.org/x/sync v0.7.0 go: downloading golang.org/x/time v0.5.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading go.uber.org/multierr v1.11.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 + set +x + tso='449575395466149889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575395466149889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575395466149889 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:13:53 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1200212004.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:13:53 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading google.golang.org/api v0.170.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 3-th time, retry later go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da table mark.finish_mark_3 not exists for 3-th check, retry later go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670b2240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5, pid:21600, start at 2024-05-06 19:13:51.757713076 +0800 CST m=+5.503838965 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:51.764 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:51.753 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:51.753 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3670b2e40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5, pid:21682, start at 2024-05-06 19:13:51.840175475 +0800 CST m=+5.525240109 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:15:51.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:13:51.851 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:03:51.851 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... go: downloading github.com/ardielle/ardielle-go v1.5.2 check diff failed 4-th time, retry later go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dolthub/maphash v0.1.0 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c table mark.finish_mark_3 not exists for 4-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:13:56 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4249f4ce-39ca-479f-8795-3a09f5bb2650 {"id":"4249f4ce-39ca-479f-8795-3a09f5bb2650","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994033} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bf2ced6 4249f4ce-39ca-479f-8795-3a09f5bb2650 /tidb/cdc/default/default/upstream/7365843217828553164 {"id":7365843217828553164,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4249f4ce-39ca-479f-8795-3a09f5bb2650 {"id":"4249f4ce-39ca-479f-8795-3a09f5bb2650","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994033} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bf2ced6 4249f4ce-39ca-479f-8795-3a09f5bb2650 /tidb/cdc/default/default/upstream/7365843217828553164 {"id":7365843217828553164,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4249f4ce-39ca-479f-8795-3a09f5bb2650 {"id":"4249f4ce-39ca-479f-8795-3a09f5bb2650","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994033} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bf2ced6 4249f4ce-39ca-479f-8795-3a09f5bb2650 /tidb/cdc/default/default/upstream/7365843217828553164 {"id":7365843217828553164,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575395466149889 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.12058.out cli changefeed create --start-ts=449575395466149889 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365843217828553164,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:13:56.761482354+08:00","start_ts":449575395466149889,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575395466149889,"checkpoint_ts":449575395466149889,"checkpoint_time":"2024-05-06 19:13:51.777"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.23081.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 9493 0 --:--:-- --:--:-- --:--:-- 9608 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:13:51.777","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:13:58.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:13:51.777","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:13:58.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:13:51.777","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:13:58.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-06 19:13:51.777' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:13:51.777","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:13:58.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:13:51.777","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:13:58.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts check diff failed 5-th time, retry later table mark.finish_mark_3 not exists for 5-th check, retry later + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 19:13:58' + echo 'sink_checkpoint_ts is 2024-05-06' 19:13:51.777 sink_checkpoint_ts is 2024-05-06 19:13:51.777 ++ date -d '2024-05-06 19:13:51.777' +%s + checkpoint_timestamp=1714994031 ++ date -d '2024-05-06 19:13:58' +%s + current_timestamp=1714994038 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + set +x + tso='449575396940709889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575396940709889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:13:58 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2312923131.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later table mark.finish_mark_3 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t1 exists + sleep 5 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:01 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68dd1b85-c816-4ec5-9880-d6746dd8f3c1 {"id":"68dd1b85-c816-4ec5-9880-d6746dd8f3c1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994039} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bff33f1 68dd1b85-c816-4ec5-9880-d6746dd8f3c1 /tidb/cdc/default/default/upstream/7365843243876038029 {"id":7365843243876038029,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68dd1b85-c816-4ec5-9880-d6746dd8f3c1 {"id":"68dd1b85-c816-4ec5-9880-d6746dd8f3c1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994039} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bff33f1 68dd1b85-c816-4ec5-9880-d6746dd8f3c1 /tidb/cdc/default/default/upstream/7365843243876038029 {"id":7365843243876038029,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68dd1b85-c816-4ec5-9880-d6746dd8f3c1 {"id":"68dd1b85-c816-4ec5-9880-d6746dd8f3c1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994039} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9bff33f1 68dd1b85-c816-4ec5-9880-d6746dd8f3c1 /tidb/cdc/default/default/upstream/7365843243876038029 {"id":7365843243876038029,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table mark.finish_mark_3 not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Mon May 6 19:14:03 CST 2024] <<<<<< START kafka consumer in capture_session_done_during_task case >>>>>> lease 22318f4d9bff33f1 revoked table mark.finish_mark_3 not exists for 8-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36716270000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7, pid:1409, start at 2024-05-06 19:14:03.051850352 +0800 CST m=+5.276752662 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:03.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:03.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:03.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_handle_key_only Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Starting Upstream TiDB... table capture_session_done_during_task.t exists check diff failed 1-th time, retry later Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36716270000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7, pid:1409, start at 2024-05-06 19:14:03.051850352 +0800 CST m=+5.276752662 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:03.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:03.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:03.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367163280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7, pid:1490, start at 2024-05-06 19:14:03.124745734 +0800 CST m=+5.295388039 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:03.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:03.132 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:03.132 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2803 0 --:--:-- --:--:-- --:--:-- 2825 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:14:04.128","puller_resolved_ts":"2024-05-06 19:13:58.378","last_synced_ts":"2024-05-06 19:13:58.427","now_ts":"2024-05-06 19:14:05.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:14:04.128","puller_resolved_ts":"2024-05-06' '19:13:58.378","last_synced_ts":"2024-05-06' '19:13:58.427","now_ts":"2024-05-06' '19:14:05.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:14:04.128","puller_resolved_ts":"2024-05-06' '19:13:58.378","last_synced_ts":"2024-05-06' '19:13:58.427","now_ts":"2024-05-06' '19:14:05.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 9-th check, retry later check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367189500017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m, pid:1354, start at 2024-05-06 19:14:05.572430994 +0800 CST m=+6.865313256 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:05.580 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:05.574 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:05.574 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367189500017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m, pid:1354, start at 2024-05-06 19:14:05.572430994 +0800 CST m=+6.865313256 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:05.580 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:05.574 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:05.574 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3671710c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m, pid:1440, start at 2024-05-06 19:14:04.012667836 +0800 CST m=+5.250549266 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:04.019 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:04.022 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:04.022 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 10-th check, retry later [Mon May 6 19:14:08 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.28512853.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2833.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 11-th check, retry later check diff failed 3-th time, retry later check diff successfully check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449575400079097857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575400079097857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:14:10 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.28712873.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:11 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3405ad27-d6f1-44b7-9d2c-6a4c7c1eb733 {"id":"3405ad27-d6f1-44b7-9d2c-6a4c7c1eb733","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994048} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c2e35ce 3405ad27-d6f1-44b7-9d2c-6a4c7c1eb733 /tidb/cdc/default/default/upstream/7365843292763854859 {"id":7365843292763854859,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3405ad27-d6f1-44b7-9d2c-6a4c7c1eb733 {"id":"3405ad27-d6f1-44b7-9d2c-6a4c7c1eb733","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994048} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c2e35ce 3405ad27-d6f1-44b7-9d2c-6a4c7c1eb733 /tidb/cdc/default/default/upstream/7365843292763854859 {"id":7365843292763854859,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3405ad27-d6f1-44b7-9d2c-6a4c7c1eb733 {"id":"3405ad27-d6f1-44b7-9d2c-6a4c7c1eb733","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994048} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c2e35ce 3405ad27-d6f1-44b7-9d2c-6a4c7c1eb733 /tidb/cdc/default/default/upstream/7365843292763854859 {"id":7365843292763854859,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: c08e6ec1-59f6-4bca-969b-18d416035b47 Info: {"upstream_id":7365843292763854859,"namespace":"default","id":"c08e6ec1-59f6-4bca-969b-18d416035b47","sink_uri":"kafka://127.0.0.1:9092/ticdc-new_ci_collation-test-18781?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:14:11.551212016+08:00","start_ts":449575399767670786,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575399767670786,"checkpoint_ts":449575399767670786,"checkpoint_time":"2024-05-06 19:14:08.186"} [Mon May 6 19:14:11 CST 2024] <<<<<< START kafka consumer in new_ci_collation case >>>>>> start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... table mark.finish_mark_3 not exists for 12-th check, retry later Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3671ce240012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514, pid:4316, start at 2024-05-06 19:14:09.956410615 +0800 CST m=+5.380926904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:09.964 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:09.929 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:09.929 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3671ce240012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514, pid:4316, start at 2024-05-06 19:14:09.956410615 +0800 CST m=+5.380926904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:09.964 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:09.929 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:09.929 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3671cfa40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-kmwcn-s8514, pid:4398, start at 2024-05-06 19:14:10.059737922 +0800 CST m=+5.434623939 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:10.070 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:10.074 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:10.074 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table new_ci_collation_test.t1 not exists for 1-th check, retry later check diff failed 4-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [2024/05/06 19:14:14.123 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 30"] table mark.finish_mark_3 not exists for 13-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:13 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5b1bec0-99c2-4095-9518-7903d05d9a4d {"id":"f5b1bec0-99c2-4095-9518-7903d05d9a4d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994051} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c2fbbd4 f5b1bec0-99c2-4095-9518-7903d05d9a4d /tidb/cdc/default/default/upstream/7365843284931286627 {"id":7365843284931286627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5b1bec0-99c2-4095-9518-7903d05d9a4d {"id":"f5b1bec0-99c2-4095-9518-7903d05d9a4d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994051} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c2fbbd4 f5b1bec0-99c2-4095-9518-7903d05d9a4d /tidb/cdc/default/default/upstream/7365843284931286627 {"id":7365843284931286627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f5b1bec0-99c2-4095-9518-7903d05d9a4d {"id":"f5b1bec0-99c2-4095-9518-7903d05d9a4d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994051} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c2fbbd4 f5b1bec0-99c2-4095-9518-7903d05d9a4d /tidb/cdc/default/default/upstream/7365843284931286627 {"id":7365843284931286627,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 3f633b46-d158-4e6c-bdd8-8907645c624b Info: {"upstream_id":7365843284931286627,"namespace":"default","id":"3f633b46-d158-4e6c-bdd8-8907645c624b","sink_uri":"kafka://127.0.0.1:9092/ticdc-tiflash-test-9929?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:14:14.131357402+08:00","start_ts":449575400079097857,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575400079097857,"checkpoint_ts":449575400079097857,"checkpoint_time":"2024-05-06 19:14:09.374"} [Mon May 6 19:14:14 CST 2024] <<<<<< START kafka consumer in tiflash case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... Verifying downstream PD is started... table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 not exists for 1-th check, retry later check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:14:14 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> [Mon May 6 19:14:15 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.58815883.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_3 not exists for 14-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367201880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:10761, start at 2024-05-06 19:14:13.262084354 +0800 CST m=+5.281024636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:13.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:13.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:13.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367201880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:10761, start at 2024-05-06 19:14:13.262084354 +0800 CST m=+5.281024636 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:13.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:13.268 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:13.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367202f40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:10833, start at 2024-05-06 19:14:13.345580639 +0800 CST m=+5.315037000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:13.352 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:13.309 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:13.309 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table cdc_tiflash_test.multi_data_type not exists for 2-th check, retry later check diff successfully table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:18 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef790c66-a9ed-46d0-af42-24cf71e86439 {"id":"ef790c66-a9ed-46d0-af42-24cf71e86439","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c46c9cc ef790c66-a9ed-46d0-af42-24cf71e86439 /tidb/cdc/default/default/upstream/7365843322200173198 {"id":7365843322200173198,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef790c66-a9ed-46d0-af42-24cf71e86439 {"id":"ef790c66-a9ed-46d0-af42-24cf71e86439","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c46c9cc ef790c66-a9ed-46d0-af42-24cf71e86439 /tidb/cdc/default/default/upstream/7365843322200173198 {"id":7365843322200173198,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef790c66-a9ed-46d0-af42-24cf71e86439 {"id":"ef790c66-a9ed-46d0-af42-24cf71e86439","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994055} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c46c9cc ef790c66-a9ed-46d0-af42-24cf71e86439 /tidb/cdc/default/default/upstream/7365843322200173198 {"id":7365843322200173198,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 7db7ec39-f221-4f48-80c6-33160d349a12 Info: {"upstream_id":7365843322200173198,"namespace":"default","id":"7db7ec39-f221-4f48-80c6-33160d349a12","sink_uri":"kafka://127.0.0.1:9092/ticdc-force_replicate_table-test-30175?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:14:18.329926572+08:00","start_ts":449575401548677122,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575401548677122,"checkpoint_ts":449575401548677122,"checkpoint_time":"2024-05-06 19:14:14.980"} [Mon May 6 19:14:18 CST 2024] <<<<<< START kafka consumer in force_replicate_table case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml table mark.finish_mark_3 not exists for 15-th check, retry later [Mon May 6 19:14:18 CST 2024] <<<<<< START cdc server in kafka_simple_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.1215412156.out server --log-file /tmp/tidb_cdc_test/kafka_simple_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_handle_key_only/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table cdc_tiflash_test.multi_data_type exists check diff successfully wait process cdc.test exit for 1-th time... table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:14:20 CST 2024] <<<<<< run test case tiflash success! >>>>>> table mark.finish_mark_3 not exists for 16-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:21 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33b98479-6d92-472f-8743-e227560268e7 {"id":"33b98479-6d92-472f-8743-e227560268e7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c5bd8ce 33b98479-6d92-472f-8743-e227560268e7 /tidb/cdc/default/default/upstream/7365843332468004665 {"id":7365843332468004665,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33b98479-6d92-472f-8743-e227560268e7 {"id":"33b98479-6d92-472f-8743-e227560268e7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c5bd8ce 33b98479-6d92-472f-8743-e227560268e7 /tidb/cdc/default/default/upstream/7365843332468004665 {"id":7365843332468004665,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/33b98479-6d92-472f-8743-e227560268e7 {"id":"33b98479-6d92-472f-8743-e227560268e7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c5bd8ce 33b98479-6d92-472f-8743-e227560268e7 /tidb/cdc/default/default/upstream/7365843332468004665 {"id":7365843332468004665,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12210.out cli tso query --pd=http://127.0.0.1:2379 table mark.finish_mark_3 not exists for 17-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later + set +x + tso='449575403328110593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575403328110593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12244.out cli changefeed create --start-ts=449575403328110593 '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-22971?protocol=simple' -c simple-handle-key-only --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/conf/changefeed.toml check diff failed 2-th time, retry later Create changefeed successfully! ID: simple-handle-key-only Info: {"upstream_id":7365843332468004665,"namespace":"default","id":"simple-handle-key-only","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-22971?protocol=simple","create_time":"2024-05-06T19:14:23.692075254+08:00","start_ts":449575403328110593,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575403328110593,"checkpoint_ts":449575403328110593,"checkpoint_time":"2024-05-06 19:14:21.768"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 18-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check diff failed 3-th time, retry later check diff failed 2-th time, retry later table mark.finish_mark_3 not exists for 19-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3672b0180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:22913, start at 2024-05-06 19:14:24.390597909 +0800 CST m=+6.464985478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:24.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:24.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:24.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3672b0180002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:22913, start at 2024-05-06 19:14:24.390597909 +0800 CST m=+6.464985478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:24.403 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:24.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:24.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3672a2cc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:22993, start at 2024-05-06 19:14:23.580473378 +0800 CST m=+5.599183362 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:23.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:23.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:23.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 exists table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later check diff failed 3-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24375.out cli tso query --pd=http://127.0.0.1:2379 table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 20-th check, retry later check diff failed 4-th time, retry later check diff failed 5-th time, retry later + set +x + tso='449575405221314561 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575405221314561 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 21-th check, retry later check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12298.out cli changefeed pause -c simple-handle-key-only ***************** properties ***************** "recordcount"="10000" "operationcount"="0" "mysql.user"="root" "mysql.host"="127.0.0.1" "insertproportion"="0" "threadcount"="10" "scanproportion"="0" "mysql.db"="move_table" "updateproportion"="0" "dotransactions"="false" "mysql.port"="4000" "requestdistribution"="uniform" "workload"="core" "readallfields"="true" "readproportion"="0" ********************************************** check diff failed 5-th time, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table force_replicate_table.t6 not exists for 2-th check, retry later table mark.finish_mark_3 not exists for 22-th check, retry later Run finished, takes 1.435158951s INSERT - Takes(s): 1.4, Count: 10000, OPS: 6989.8, Avg(us): 1390, Min(us): 778, Max(us): 7414, 95th(us): 3000, 99th(us): 3000 [Mon May 6 19:14:33 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2445324455.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12327.out cli changefeed update -c simple-handle-key-only '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-22971?protocol=simple&max-message-bytes=700' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/conf/changefeed.toml --no-confirm check diff successfully check diff failed 1-th time, retry later Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/simple-handle-key-only-22971?protocol=simple To:kafka://127.0.0.1:9092/simple-handle-key-only-22971?protocol=simple&max-message-bytes=700} {Type:update Path:[Config SyncPointInterval] From: To:0xc00114a898} {Type:update Path:[Config SyncPointRetention] From: To:0xc00114a8b8} {Type:update Path:[Config Consistent] From: To:0xc00179c230} table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 Update changefeed config successfully! ID: simple-handle-key-only Info: {"upstream_id":7365843332468004665,"namespace":"default","id":"simple-handle-key-only","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-22971?protocol=simple\u0026max-message-bytes=700","create_time":"2024-05-06T19:14:23.692075254+08:00","start_ts":449575403328110593,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":0,"checkpoint_ts":449575405753204739,"checkpoint_time":"2024-05-06 19:14:31.019"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 23-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 check diff successfully run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 check diff failed 1-th time, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12359.out cli changefeed resume -c simple-handle-key-only Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 24-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:36 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/667dce24-5054-4528-b99d-e95726294f19 {"id":"667dce24-5054-4528-b99d-e95726294f19","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994073} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbefb 667dce24-5054-4528-b99d-e95726294f19 /tidb/cdc/default/default/upstream/7365843371209193735 {"id":7365843371209193735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/667dce24-5054-4528-b99d-e95726294f19 {"id":"667dce24-5054-4528-b99d-e95726294f19","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994073} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbefb 667dce24-5054-4528-b99d-e95726294f19 /tidb/cdc/default/default/upstream/7365843371209193735 {"id":7365843371209193735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/667dce24-5054-4528-b99d-e95726294f19 {"id":"667dce24-5054-4528-b99d-e95726294f19","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994073} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbefb 667dce24-5054-4528-b99d-e95726294f19 /tidb/cdc/default/default/upstream/7365843371209193735 {"id":7365843371209193735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24507.out cli changefeed create --start-ts=449575405221314561 '--sink-uri=kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:14:26.301 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/05/06 19:14:26.479 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:14:26.493 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:14:26.943 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:14:26.944 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:14:26.956 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:14:26.957 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:14:27.390 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:14:27.404 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:14:27.837 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:14:27.839 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:14:27.852 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 19:14:27.854 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:14:28.282 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:14:28.299 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:14:28.732 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:14:28.742 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:14:28.750 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:14:28.755 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 19:14:29.184 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 19:14:29.201 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:14:29.642 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:14:29.657 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:14:29.661 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:14:29.667 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 19:14:30.109 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 19:14:30.119 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 19:14:30.582 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:14:30.588 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:14:30.591 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 19:14:30.606 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:14:31.029 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 19:14:31.055 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [mysql] 2024/05/06 19:14:31 connection.go:299: invalid connection [2024/05/06 19:14:36.381 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2024/05/06 19:14:36.655 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:14:36.715 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] Create changefeed successfully! ID: 3c3cd017-77e7-4af4-9017-6b6eb70f0767 Info: {"upstream_id":7365843371209193735,"namespace":"default","id":"3c3cd017-77e7-4af4-9017-6b6eb70f0767","sink_uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:14:36.630238201+08:00","start_ts":449575405221314561,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575405221314561,"checkpoint_ts":449575405221314561,"checkpoint_time":"2024-05-06 19:14:28.990"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 wait process cdc.test exit for 2-th time... [2024/05/06 19:14:36.829 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:14:36.834 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:14:36.892 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:14:36.900 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:14:36.998 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:14:37.069 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:14:37.164 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:14:37.177 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:14:37 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> [2024/05/06 19:14:37.335 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:14:37.337 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:14:37.346 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:14:37.506 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:14:37.522 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:14:37.535 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 check diff failed 2-th time, retry later [2024/05/06 19:14:37.704 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:14:37.758 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:14:37.774 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:14:37.901 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:14:37.919 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 19:14:37.937 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] + set +x [2024/05/06 19:14:38.091 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 19:14:38.119 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:14:38.138 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:14:38.280 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:14:38.297 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 19:14:38.297 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] table mark.finish_mark_3 not exists for 25-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Mon May 6 19:14:38 CST 2024] <<<<<< START kafka consumer in move_table case >>>>>> [Mon May 6 19:14:38 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2454724549.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x [2024/05/06 19:14:38.469 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 19:14:38.537 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:14:38.557 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 1-th check, retry later [2024/05/06 19:14:38.653 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:14:38.668 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 19:14:38.699 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 19:14:38.850 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 19:14:38.879 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 19:14:38.898 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:14:39.040 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 19:14:39.040 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 19:14:39.057 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=7,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later [2024/05/06 19:14:39.206 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:14:39.223 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 19:14:39.231 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 19:14:39.369 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 19:14:39.399 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 19:14:39.419 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:14:39.540 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 19:14:39.569 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 19:14:39.574 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 19:14:39.751 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 19:14:39.771 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 19:14:39.801 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 19:14:39.925 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 19:14:39.993 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 19:14:40.034 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] check diff failed 3-th time, retry later [2024/05/06 19:14:40.120 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 19:14:40.144 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 19:14:40.185 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 19:14:40.316 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 2-th check, retry later [2024/05/06 19:14:40.383 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 19:14:40.429 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 19:14:40.487 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 19:14:40.512 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 19:14:40.558 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 26-th check, retry later [2024/05/06 19:14:40.651 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 19:14:40.731 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 19:14:40.774 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 19:14:40.818 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 19:14:40.843 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 19:14:40.907 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 19:14:41.015 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 19:14:41.110 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 19:14:41.161 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 19:14:41.180 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 19:14:41.205 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/06 19:14:41.275 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 19:14:41.347 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:41 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {UpstreamID:7365843371209193735 Namespace:default ID:3c3cd017-77e7-4af4-9017-6b6eb70f0767 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:14:36.630238201 +0800 CST StartTs:449575405221314561 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001412fc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575407213608965} {CheckpointTs:449575405968425008 MinTableBarrierTs:449575408196911112 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/667dce24-5054-4528-b99d-e95726294f19 {"id":"667dce24-5054-4528-b99d-e95726294f19","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994073} /tidb/cdc/default/__cdc_meta__/capture/f666d569-91df-4969-850a-71718faf61e1 {"id":"f666d569-91df-4969-850a-71718faf61e1","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbefb 667dce24-5054-4528-b99d-e95726294f19 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbf55 f666d569-91df-4969-850a-71718faf61e1 /tidb/cdc/default/default/changefeed/info/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"upstream-id":7365843371209193735,"namespace":"default","changefeed-id":"3c3cd017-77e7-4af4-9017-6b6eb70f0767","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:14:36.630238201+08:00","start-ts":449575405221314561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575407213608965} /tidb/cdc/default/default/changefeed/status/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":449575405981532330,"min-table-barrier-ts":449575408196911112,"admin-job-type":0} /tidb/cdc/default/default/task/position/667dce24-5054-4528-b99d-e95726294f19/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f666d569-91df-4969-850a-71718faf61e1/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843371209193735 {"id":7365843371209193735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {UpstreamID:7365843371209193735 Namespace:default ID:3c3cd017-77e7-4af4-9017-6b6eb70f0767 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:14:36.630238201 +0800 CST StartTs:449575405221314561 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001412fc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575407213608965} {CheckpointTs:449575405968425008 MinTableBarrierTs:449575408196911112 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/667dce24-5054-4528-b99d-e95726294f19 {"id":"667dce24-5054-4528-b99d-e95726294f19","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994073} /tidb/cdc/default/__cdc_meta__/capture/f666d569-91df-4969-850a-71718faf61e1 {"id":"f666d569-91df-4969-850a-71718faf61e1","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbefb 667dce24-5054-4528-b99d-e95726294f19 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbf55 f666d569-91df-4969-850a-71718faf61e1 /tidb/cdc/default/default/changefeed/info/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"upstream-id":7365843371209193735,"namespace":"default","changefeed-id":"3c3cd017-77e7-4af4-9017-6b6eb70f0767","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:14:36.630238201+08:00","start-ts":449575405221314561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575407213608965} /tidb/cdc/default/default/changefeed/status/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":449575405981532330,+ grep -q 'failed to get info:' "min-table-barrier-ts":449575408196911112,"admin-job-type":0} /tidb/cdc/default/default/task/position/667dce24-5054-4528-b99d-e95726294f19/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f666d569-91df-4969-850a-71718faf61e1/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843371209193735 {"id":7365843371209193735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {UpstreamID:7365843371209193735 Namespace:default ID:3c3cd017-77e7-4af4-9017-6b6eb70f0767 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:14:36.630238201 +0800 CST StartTs:449575405221314561 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001412fc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575407213608965} {CheckpointTs:449575405968425008 MinTableBarrierTs:449575408196911112 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/667dce24-5054-4528-b99d-e95726294f19 {"id":"667dce24-5054-4528-b99d-e95726294f19","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994073} /tidb/cdc/default/__cdc_meta__/capture/f666d569-91df-4969-850a-71718faf61e1 {"id":"f666d569-91df-4969-850a-71718faf61e1","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbefb 667dce24-5054-4528-b99d-e95726294f19 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbf55 f666d569-91df-4969-850a-71718faf61e1 /tidb/cdc/default/default/changefeed/info/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"upstream-id":7365843371209193735,"namespace":"default","changefeed-id":"3c3cd017-77e7-4af4-9017-6b6eb70f0767","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:14:36.630238201+08:00","start-ts":449575405221314561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575407213608965} /tidb/cdc/default/default/changefeed/status/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":449575405981532330,"min-table-barrier-ts":449575408196911112,"admin-job-type":0} /tidb/cdc/default/default/task/position/667dce24-5054-4528-b99d-e95726294f19/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f666d569-91df-4969-850a-71718faf61e1/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843371209193735 {"id":7365843371209193735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Mon May 6 19:14:41 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2466624668.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 check diff failed 4-th time, retry later run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 27-th check, retry later wait process cdc.test exit for 2-th time... table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:14:43 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> wait process cdc.test exit for 2-th time... check diff failed 5-th time, retry later wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:14:44 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3673b7500003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5, pid:24664, start at 2024-05-06 19:14:41.237568898 +0800 CST m=+5.214281266 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:41.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:41.236 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:41.236 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3673b7500003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5, pid:24664, start at 2024-05-06 19:14:41.237568898 +0800 CST m=+5.214281266 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:41.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:41.236 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:41.236 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3673b7e80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-qfjm4-dzrj5, pid:24742, start at 2024-05-06 19:14:41.300874781 +0800 CST m=+5.229605954 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:41.308 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:41.274 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:41.274 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_3 not exists for 28-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {UpstreamID:7365843371209193735 Namespace:default ID:3c3cd017-77e7-4af4-9017-6b6eb70f0767 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:14:36.630238201 +0800 CST StartTs:449575405221314561 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00301e900 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575407213608965} {CheckpointTs:449575408996450307 MinTableBarrierTs:449575408996450308 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15d3204a-7e4d-4f20-9e52-f157075a28eb {"id":"15d3204a-7e4d-4f20-9e52-f157075a28eb","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994081} /tidb/cdc/default/__cdc_meta__/capture/667dce24-5054-4528-b99d-e95726294f19 {"id":"667dce24-5054-4528-b99d-e95726294f19","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994073} /tidb/cdc/default/__cdc_meta__/capture/f666d569-91df-4969-850a-71718faf61e1 {"id":"f666d569-91df-4969-850a-71718faf61e1","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbefb 667dce24-5054-4528-b99d-e95726294f19 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbf55 f666d569-91df-4969-850a-71718faf61e1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbf85 15d3204a-7e4d-4f20-9e52-f157075a28eb /tidb/cdc/default/default/changefeed/info/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"upstream-id":7365843371209193735,"namespace":"default","changefeed-id":"3c3cd017-77e7-4af4-9017-6b6eb70f0767","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:14:36.630238201+08:00","start-ts":449575405221314561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575407213608965} /tidb/cdc/default/default/changefeed/status/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":449575408996450307,"min-table-barrier-ts":449575408996450308,"admin-job-type":0} /tidb/cdc/default/default/task/position/15d3204a-7e4d-4f20-9e52-f157075a28eb/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/667dce24-5054-4528-b99d-e95726294f19/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f666d569-91df-4969-850a-71718faf61e1/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843371209193735 {"id":7365843371209193735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {UpstreamID:7365843371209193735 Namespace:default ID:3c3cd017-77e7-4af4-9017-6b6eb70f0767 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:14:36.630238201 +0800 CST StartTs:449575405221314561 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00301e900 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575407213608965} {CheckpointTs:449575408996450307 MinTableBarrierTs:449575408996450308 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15d3204a-7e4d-4f20-9e52-f157075a28eb {"id":"15d3204a-7e4d-4f20-9e52-f157075a28eb","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994081} /tidb/cdc/default/__cdc_meta__/capture/667dce24-5054-4528-b99d-e95726294f19 {"id":"667dce24-5054-4528-b99d-e95726294f19","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994073} /tidb/cdc/default/__cdc_meta__/capture/f666d569-91df-4969-850a-71718faf61e1 {"id":"f666d569-91df-4969-850a-71718faf61e1","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbefb 667dce24-5054-4528-b99d-e95726294f19 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbf55 f666d569-91df-4969-850a-71718faf61e1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbf85 15d3204a-7e4d-4f20-9e52-f157075a28eb /tidb/cdc/default/default/changefeed/info/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"upstream-id":7365843371209193735,"namespace":"default","changefeed-id":"3c3cd017-77e7-4af4-9017-6b6eb70f0767","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:14:36.630238201+08:00","start-ts":449575405221314561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575407213608965} /tidb/cdc/default/default/changefeed/status/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":449575408996450307,"min-table-barrier-ts":449575408996450308,"admin-job-type":0} /tidb/cdc/default/default/task/position/15d3204a-7e4d-4f20-9e52-f157075a28eb/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/667dce24-5054-4528-b99d-e95726294f19/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f666d569-91df-4969-850a-71718faf61e1/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843371209193735 {"id":7365843371209193735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {UpstreamID:7365843371209193735 Namespace:default ID:3c3cd017-77e7-4af4-9017-6b6eb70f0767 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:14:36.630238201 +0800 CST StartTs:449575405221314561 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00301e900 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575407213608965} {CheckpointTs:449575408996450307 MinTableBarrierTs:449575408996450308 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15d3204a-7e4d-4f20-9e52-f157075a28eb {"id":"15d3204a-7e4d-4f20-9e52-f157075a28eb","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994081} /tidb/cdc/default/__cdc_meta__/capture/667dce24-5054-4528-b99d-e95726294f19 {"id":"667dce24-5054-4528-b99d-e95726294f19","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994073} /tidb/cdc/default/__cdc_meta__/capture/f666d569-91df-4969-850a-71718faf61e1 {"id":"f666d569-91df-4969-850a-71718faf61e1","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbefb 667dce24-5054-4528-b99d-e95726294f19 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbf55 f666d569-91df-4969-850a-71718faf61e1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9c7cbf85 15d3204a-7e4d-4f20-9e52-f157075a28eb /tidb/cdc/default/default/changefeed/info/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"upstream-id":7365843371209193735,"namespace":"default","changefeed-id":"3c3cd017-77e7-4af4-9017-6b6eb70f0767","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-27193?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:14:36.630238201+08:00","start-ts":449575405221314561,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-thresh+ grep -q 'etcd info' old":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575407213608965} /tidb/cdc/default/default/changefeed/status/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":449575408996450307,"min-table-barrier-ts":449575408996450308,"admin-job-type":0} /tidb/cdc/default/default/task/position/15d3204a-7e4d-4f20-9e52-f157075a28eb/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/667dce24-5054-4528-b99d-e95726294f19/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f666d569-91df-4969-850a-71718faf61e1/3c3cd017-77e7-4af4-9017-6b6eb70f0767 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843371209193735 {"id":7365843371209193735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table move_table.usertable exists go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/xdg/scram v1.0.5 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_3 not exists for 29-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... [2024/05/06 19:14:46.574 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] go: downloading github.com/ardielle/ardielle-go v1.5.2 [2024/05/06 19:14:46.759 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:14:46.767 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.26087.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 19:14:46.921 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:14:46.921 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:14:46.931 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:14:46.933 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:14:47.086 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:14:47.096 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] wait process cdc.test exit for 2-th time... [2024/05/06 19:14:47.255 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:14:47.256 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:14:47.260 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:14:47.261 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:14:47.418 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:14:47.423 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:14:47.579 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:14:47.580 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:14:47.587 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:14:47.589 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] wait process cdc.test exit for 3-th time... [2024/05/06 19:14:47.750 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 19:14:47.757 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:14:47.920 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:14:47.921 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:14:47.922 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:14:47.922 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 19:14:48.080 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 19:14:48.093 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:14:48 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> + set +x + tso='449575409847369730 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575409847369730 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:14:48 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2611826120.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:14:48.239 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:14:48.242 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 19:14:48.252 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:14:48.252 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:14:48.394 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] table mark.finish_mark_3 not exists for 30-th check, retry later [2024/05/06 19:14:48.412 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 19:14:48.551 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:14:48.553 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 19:14:48.574 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:14:48.575 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 19:14:48.715 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 19:14:48.743 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 19:14:48.880 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 19:14:48.882 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 19:14:48.909 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 19:14:48.910 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:14:49.046 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 19:14:49.075 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 19:14:49.201 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 19:14:49.203 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:14:49.235 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 19:14:49.236 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 19:14:49.365 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 19:14:49.401 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 19:14:49.539 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 19:14:49.540 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 19:14:49.572 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 19:14:49.573 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/06 19:14:49.700 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 19:14:49.732 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 19:14:49.874 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 19:14:49.876 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 19:14:49.906 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 19:14:49.907 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:14:50.048 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/06 19:14:50.075 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] table mark.finish_mark_3 not exists for 31-th check, retry later [2024/05/06 19:14:50.225 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 19:14:50.227 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 19:14:50.247 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 19:14:50.248 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 19:14:50.393 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 19:14:50.416 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 19:14:50.551 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 19:14:50.553 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 19:14:50.576 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 19:14:50.577 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 19:14:50.722 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 19:14:50.738 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 19:14:50.893 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 19:14:50.895 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/06 19:14:50.900 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 19:14:50.900 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 19:14:51.068 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/05/06 19:14:51.070 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/06 19:14:51.254 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/05/06 19:14:51.254 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/06 19:14:51.257 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/06 19:14:51.259 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/06 19:14:51.414 +08:00] [INFO] [main.go:220] ["0 insert success: 2900"] [2024/05/06 19:14:51.425 +08:00] [INFO] [main.go:220] ["1 insert success: 2900"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:51 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/532b1831-40a0-4e04-9db5-f7d0dfcf6446 {"id":"532b1831-40a0-4e04-9db5-f7d0dfcf6446","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994088} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9cc39dd4 532b1831-40a0-4e04-9db5-f7d0dfcf6446 /tidb/cdc/default/default/upstream/7365843455851026675 {"id":7365843455851026675,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/532b1831-40a0-4e04-9db5-f7d0dfcf6446 {"id":"532b1831-40a0-4e04-9db5-f7d0dfcf6446","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994088} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9cc39dd4 532b1831-40a0-4e04-9db5-f7d0dfcf6446 /tidb/cdc/default/default/upstream/7365843455851026675 {"id":7365843455851026675,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/532b1831-40a0-4e04-9db5-f7d0dfcf6446 {"id":"532b1831-40a0-4e04-9db5-f7d0dfcf6446","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994088} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9cc39dd4 532b1831-40a0-4e04-9db5-f7d0dfcf6446 /tidb/cdc/default/default/upstream/7365843455851026675 {"id":7365843455851026675,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.26178.out cli changefeed create --start-ts=449575409847369730 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-15614?protocol=open-protocol&partition-num=4&kafka-version=2.4.1' [2024/05/06 19:14:51.572 +08:00] [INFO] [main.go:234] ["0 delete success: 1500"] [2024/05/06 19:14:51.573 +08:00] [INFO] [main.go:220] ["0 insert success: 3000"] Create changefeed successfully! ID: 8d23fd49-126e-49e1-866d-7e118afb412a Info: {"upstream_id":7365843455851026675,"namespace":"default","id":"8d23fd49-126e-49e1-866d-7e118afb412a","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-15614?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1","create_time":"2024-05-06T19:14:51.678803927+08:00","start_ts":449575409847369730,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575409847369730,"checkpoint_ts":449575409847369730,"checkpoint_time":"2024-05-06 19:14:46.637"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 32-th check, retry later + set +x [Mon May 6 19:14:53 CST 2024] <<<<<< START kafka consumer in ddl_attributes case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 33-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36746870000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m, pid:4152, start at 2024-05-06 19:14:52.586563976 +0800 CST m=+5.300453302 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:52.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:52.572 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:52.572 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36746870000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m, pid:4152, start at 2024-05-06 19:14:52.586563976 +0800 CST m=+5.300453302 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:52.596 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:52.572 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:52.572 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36746aa40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m, pid:4230, start at 2024-05-06 19:14:52.732928258 +0800 CST m=+5.398037623 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:16:52.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:14:52.713 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:04:52.713 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh: line 1: 12385 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 34-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:14:56.770 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/05/06 19:14:56.951 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:14:56.961 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:14:57.113 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:14:57.115 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:14:57.134 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:14:57.135 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:14:57.281 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:14:57.300 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:14:57.446 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:14:57.447 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:14:57.458 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 19:14:57.459 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:14:57.615 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:14:57.628 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:14:57.785 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:14:57.786 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:14:57.801 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 19:14:57.802 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:14:57 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.56895691.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 pass check, checkpoint tso not forward after 10s run task successfully wait process 8789 exit for 1-th time... wait process 8789 exit for 2-th time... wait process 8789 exit for 3-th time... wait process 8789 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8789) - No such process wait process 8789 exit for 5-th time... process 8789 already exit [Mon May 6 19:14:53 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.94999501.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:14:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843109605650266 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:13:27.750069952 +0800 CST StartTs:449575389127507973 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003838360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575389166567426} {CheckpointTs:449575393072513034 MinTableBarrierTs:449575412038631435 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/382a9eb0-4a51-44ab-9c5c-2574fd9dcb61 {"id":"382a9eb0-4a51-44ab-9c5c-2574fd9dcb61","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994093} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b8c8ace 382a9eb0-4a51-44ab-9c5c-2574fd9dcb61 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843109605650266,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:13:27.750069952+08:00","start-ts":449575389127507973,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575389166567426} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575393072513034,"min-table-barrier-ts":449575412301037579,"admin-job-type":0} /tidb/cdc/default/default/task/position/382a9eb0-4a51-44ab-9c5c-2574fd9dcb61/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843109605650266 {"id":7365843109605650266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843109605650266 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:13:27.750069952 +0800 CST StartTs:449575389127507973 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003838360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575389166567426} {CheckpointTs:449575393072513034 MinTableBarrierTs:449575412038631435 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/382a9eb0-4a51-44ab-9c5c-2574fd9dcb61 {"id":"382a9eb0-4a51-44ab-9c5c-2574fd9dcb61","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994093} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b8c8ace 382a9eb0-4a51-44ab-9c5c-2574fd9dcb61 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843109605650266,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:13:27.750069952+08:00","start-ts":449575389127507973,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-chec+ grep -q 'failed to get info:' k-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575389166567426} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575393072513034,"min-table-barrier-ts":449575412301037579,"admin-job-type":0} /tidb/cdc/default/default/task/position/382a9eb0-4a51-44ab-9c5c-2574fd9dcb61/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843109605650266 {"id":7365843109605650266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365843109605650266 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-06 19:13:27.750069952 +0800 CST StartTs:449575389127507973 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003838360 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575389166567426} {CheckpointTs:449575393072513034 MinTableBarrierTs:449575412038631435 AdminJobType:noop} span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449575393072513034, checkpointTs: 449575393072513034, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/382a9eb0-4a51-44ab-9c5c-2574fd9dcb61 {"id":"382a9eb0-4a51-44ab-9c5c-2574fd9dcb61","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994093} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9b8c8ace 382a9eb0-4a51-44ab-9c5c-2574fd9dcb61 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365843109605650266,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-29808?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-06T19:13:27.750069952+08:00","start-ts":449575389127507973,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575389166567426} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449575393072513034,"min-table-barrier-ts":449575412301037579,"admin-job-type":0} /tidb/cdc/default/default/task/position/382a9eb0-4a51-44ab-9c5c-2574fd9dcb61/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365843109605650266 {"id":7365843109605650266,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later [2024/05/06 19:14:57.950 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 19:14:57.969 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:14:58.114 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:14:58.116 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:14:58.124 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 19:14:58.125 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] check diff successfully check_ts_forward ddl-only-block-related-table [2024/05/06 19:14:58.274 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 19:14:58.283 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] table mark.finish_mark_3 not exists for 35-th check, retry later [2024/05/06 19:14:58.438 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:14:58.440 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:14:58.449 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 19:14:58.449 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:14:58.590 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 19:14:58.606 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 19:14:58.743 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 19:14:58.745 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:14:58.760 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 19:14:58.761 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:14:58.897 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 19:14:58.911 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 19:14:59.058 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:14:59.060 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 19:14:59.068 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 19:14:59.069 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 19:14:59.223 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 19:14:59.230 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 19:14:59.379 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 19:14:59.381 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 19:14:59.392 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:14:59.393 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 19:14:59.543 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 19:14:59.562 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/06 19:14:59.714 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 19:14:59.716 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 19:14:59.735 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 19:14:59.736 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 19:14:59.879 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 19:14:59.905 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 19:15:00.036 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 19:15:00.037 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 19:15:00.065 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 19:15:00.065 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 19:15:00.208 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/06 19:15:00.224 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 36-th check, retry later [2024/05/06 19:15:00.367 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 19:15:00.369 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 19:15:00.383 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 19:15:00.384 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [2024/05/06 19:15:00.524 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 19:15:00.542 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 19:15:00.681 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/06 19:15:00.683 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 19:15:00.703 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 19:15:00.704 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:00 GMT < Content-Length: 859 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14f434c1-b62d-4ee3-9e1b-4e10004f625f {"id":"14f434c1-b62d-4ee3-9e1b-4e10004f625f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994097} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9cf36830 14f434c1-b62d-4ee3-9e1b-4e10004f625f /tidb/cdc/default/default/upstream/7365843505187590679 {"id":7365843505187590679,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14f434c1-b62d-4ee3-9e1b-4e10004f625f {"id":"14f434c1-b62d-4ee3-9e1b-4e10004f625f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994097} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9cf36830 14f434c1-b62d-4ee3-9e1b-4e10004f625f /tidb/cdc/default/default/upstream/7365843505187590679 {"id":7365843505187590679,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14f434c1-b62d-4ee3-9e1b-4e10004f625f {"id":"14f434c1-b62d-4ee3-9e1b-4e10004f625f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994097} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9cf36830 14f434c1-b62d-4ee3-9e1b-4e10004f625f /tidb/cdc/default/default/upstream/7365843505187590679 {"id":7365843505187590679,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x changefeed is working normally rts: 449575412825325600->449575413087207400 checkpoint: 449575412825325600->449575413349351400 run task successfully [2024/05/06 19:14:55.495 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`many_cols`] [where="((TRUE) AND TRUE)"] ["source checksum"=4115266968] ["target checksum"=3270617779] ["get source checksum cost"=1.589216ms] ["get target checksum cost"=1.478819ms] [2024/05/06 19:14:55.497 +08:00] [WARN] [diff.go:893] ["find different row"] [column=val] [row1="{ id: 1, val: 1980, }"] [row2="{ id: 1, val: 1929, }"] [2024/05/06 19:14:55.584 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`many_cols`] [where="((TRUE) AND TRUE)"] [cost=88.966707ms] [2024/05/06 19:14:55.586 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] [2024/05/06 19:15:00.838 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 19:15:00.863 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] Create changefeed successfully! ID: 7c40fd1a-350c-4417-8d61-8d2d752d2562 Info: {"upstream_id":7365843505187590679,"namespace":"default","id":"7c40fd1a-350c-4417-8d61-8d2d752d2562","sink_uri":"kafka://127.0.0.1:9092/ticdc-region-merge-test-17553?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:15:00.911522688+08:00","start_ts":449575413539667973,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575413539667973,"checkpoint_ts":449575413539667973,"checkpoint_time":"2024-05-06 19:15:00.722"} [Mon May 6 19:15:00 CST 2024] <<<<<< START kafka consumer in region_merge case >>>>>> wait process cdc.test exit for 1-th time... [2024/05/06 19:15:01.031 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/06 19:15:01.033 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 19:15:01.041 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/06 19:15:01.042 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 19:15:01.194 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/06 19:15:01.198 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/05/06 19:15:01.352 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/06 19:15:01.354 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/05/06 19:15:01.362 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/06 19:15:01.363 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/06 19:15:01.512 +08:00] [INFO] [main.go:220] ["0 insert success: 2900"] [2024/05/06 19:14:59.003 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/06 19:14:59.006 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/06 19:14:59.007 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=667dce24-5054-4528-b99d-e95726294f19] [2024/05/06 19:14:59.007 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/06 19:14:59.008 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/06 19:14:59.194 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=3c3cd017-77e7-4af4-9017-6b6eb70f0767] [captureID=15d3204a-7e4d-4f20-9e52-f157075a28eb] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 19:14:59.393 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=3c3cd017-77e7-4af4-9017-6b6eb70f0767] [captureID=667dce24-5054-4528-b99d-e95726294f19] [processorDetail="{\"table_ids\":[108,106]}"] [2024/05/06 19:14:59.594 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=3c3cd017-77e7-4af4-9017-6b6eb70f0767] [captureID=f666d569-91df-4969-850a-71718faf61e1] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 19:14:59.594 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"15d3204a-7e4d-4f20-9e52-f157075a28eb\":[],\"667dce24-5054-4528-b99d-e95726294f19\":[{\"ID\":108,\"Changefeed\":\"3c3cd017-77e7-4af4-9017-6b6eb70f0767\"},{\"ID\":106,\"Changefeed\":\"3c3cd017-77e7-4af4-9017-6b6eb70f0767\"}],\"f666d569-91df-4969-850a-71718faf61e1\":[]}"] [2024/05/06 19:14:59.594 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=3c3cd017-77e7-4af4-9017-6b6eb70f0767&target-cp-id=f666d569-91df-4969-850a-71718faf61e1&table-id=108"] [2024/05/06 19:14:59.643 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/06 19:14:59.643 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=3c3cd017-77e7-4af4-9017-6b6eb70f0767&target-cp-id=f666d569-91df-4969-850a-71718faf61e1&table-id=106"] [2024/05/06 19:14:59.693 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/05/06 19:14:59.693 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=667dce24-5054-4528-b99d-e95726294f19] [targetCapture=f666d569-91df-4969-850a-71718faf61e1] table move_table.check1 exists check diff successfully \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } wait process cdc.test exit for 2-th time... Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [2024/05/06 19:15:01.533 +08:00] [INFO] [main.go:220] ["1 insert success: 2900"] [2024/05/06 19:15:01.683 +08:00] [INFO] [main.go:220] ["0 insert success: 3000"] [2024/05/06 19:15:01.684 +08:00] [INFO] [main.go:234] ["0 delete success: 1500"] [2024/05/06 19:15:01.708 +08:00] [INFO] [main.go:234] ["1 delete success: 1500"] [2024/05/06 19:15:01.708 +08:00] [INFO] [main.go:220] ["1 insert success: 3000"] [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } split_and_random_merge scale: 20 [Pipeline] // withCredentials [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:15:02 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> [Pipeline] // timeout [Pipeline] } [Pipeline] // stage ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate table mark.finish_mark_3 not exists for 37-th check, retry later [Pipeline] } [Pipeline] // withEnv [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36750f800003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7, pid:4477, start at 2024-05-06 19:15:03.26670878 +0800 CST m=+5.357421231 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:03.275 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:03.264 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:03.264 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 38-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test1.finishmark exists [2024/05/06 19:15:00.190 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/05/06 19:15:00.192 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/05/06 19:15:00.197 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/05/06 19:15:00.202 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/05/06 19:15:00.204 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/05/06 19:15:00.204 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=20.705286ms] [2024/05/06 19:15:00.207 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/05/06 19:15:00.209 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/05/06 19:15:00.211 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/05/06 19:15:00.216 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/05/06 19:15:00.218 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/05/06 19:15:00.218 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=13.35742ms] [2024/05/06 19:15:00.218 +08:00] [INFO] [main.go:95] ["compare checksum passed"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:15:01 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> [2024/05/06 19:15:05.760 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/06 19:15:05.762 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/06 19:15:05.762 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=667dce24-5054-4528-b99d-e95726294f19] [2024/05/06 19:15:05.763 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/06 19:15:05.763 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/06 19:15:05.943 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=3c3cd017-77e7-4af4-9017-6b6eb70f0767] [captureID=15d3204a-7e4d-4f20-9e52-f157075a28eb] [processorDetail="{\"table_ids\":[]}"] [2024/05/06 19:15:06.143 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=3c3cd017-77e7-4af4-9017-6b6eb70f0767] [captureID=667dce24-5054-4528-b99d-e95726294f19] [processorDetail="{\"table_ids\":[110]}"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36750f800003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7, pid:4477, start at 2024-05-06 19:15:03.26670878 +0800 CST m=+5.357421231 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:03.275 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:03.264 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:03.264 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367510240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7, pid:4556, start at 2024-05-06 19:15:03.333788344 +0800 CST m=+5.373433511 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:03.340 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:03.305 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:03.305 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/06 19:15:06.343 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=3c3cd017-77e7-4af4-9017-6b6eb70f0767] [captureID=f666d569-91df-4969-850a-71718faf61e1] [processorDetail="{\"table_ids\":[108]}"] [2024/05/06 19:15:06.343 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"15d3204a-7e4d-4f20-9e52-f157075a28eb\":[],\"667dce24-5054-4528-b99d-e95726294f19\":[{\"ID\":110,\"Changefeed\":\"3c3cd017-77e7-4af4-9017-6b6eb70f0767\"}],\"f666d569-91df-4969-850a-71718faf61e1\":[{\"ID\":108,\"Changefeed\":\"3c3cd017-77e7-4af4-9017-6b6eb70f0767\"}]}"] [2024/05/06 19:15:06.343 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=3c3cd017-77e7-4af4-9017-6b6eb70f0767&target-cp-id=f666d569-91df-4969-850a-71718faf61e1&table-id=110"] [2024/05/06 19:15:06.394 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=110] [2024/05/06 19:15:06.394 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=667dce24-5054-4528-b99d-e95726294f19] [targetCapture=f666d569-91df-4969-850a-71718faf61e1] check diff successfully table mark.finish_mark_3 not exists for 39-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:15:06.999 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] table move_table.check2 not exists for 1-th check, retry later [2024/05/06 19:15:07.189 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:15:07.191 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:15:07.361 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/06 19:15:07.361 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:15:07.361 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:15:07.361 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:15:07.529 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:15:07.529 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/06 19:15:07.692 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 19:15:07.692 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:15:07.693 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:15:07.693 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/06 19:15:07.860 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:15:07.864 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:15:08.025 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:15:08.025 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/06 19:15:08.025 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:15:08.026 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:15:08.181 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:15:08.186 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 19:15:08.331 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/06 19:15:08.332 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:15:08.337 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:15:08.338 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:15:08.486 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/06 19:15:08.499 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] *************************** 1. row *************************** count(distinct region_id): 1 [2024/05/06 19:15:08.640 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 19:15:08.641 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:15:08.657 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:15:08.658 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] table mark.finish_mark_3 not exists for 40-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:15:08.796 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/06 19:15:08.828 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 19:15:08.956 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/06 19:15:08.957 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:15:09.000 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:15:09.000 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] table move_table.check2 exists Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore [2024/05/06 19:15:09.121 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/06 19:15:09.160 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 19:15:09.285 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/06 19:15:09.286 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] check diff successfully Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:15:09.315 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 19:15:09.315 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:15:09.444 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 19:15:09.475 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [Mon May 6 19:15:09 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.59015903.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/06 19:15:09.611 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:15:09.612 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 19:15:09.639 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 19:15:09.640 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/06 19:15:09.788 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] wait process cdc.test exit for 1-th time... [2024/05/06 19:15:09.802 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 19:15:09.939 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/06 19:15:09.940 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 19:15:09.952 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 19:15:09.953 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] wait process cdc.test exit for 2-th time... [2024/05/06 19:15:10.109 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/06 19:15:10.113 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/06 19:15:10.271 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/06 19:15:10.272 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/06 19:15:10.276 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 19:15:10.277 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/06 19:15:10.434 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/06 19:15:10.435 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] table mark.finish_mark_3 not exists for 41-th check, retry later wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36757e100013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:13530, start at 2024-05-06 19:15:10.36713147 +0800 CST m=+5.771303329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:10.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:10.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:10.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/06 19:15:10.590 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/06 19:15:10.591 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/06 19:15:10.597 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/06 19:15:10.597 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/06 19:15:10.748 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/06 19:15:10.754 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/06 19:15:10.911 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] [2024/05/06 19:15:10.912 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/06 19:15:10.917 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/06 19:15:10.918 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:15:11 CST 2024] <<<<<< run test case move_table success! >>>>>> [2024/05/06 19:15:11.076 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/06 19:15:11.082 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/06 19:15:11.238 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] [2024/05/06 19:15:11.239 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/06 19:15:11.248 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/06 19:15:11.248 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_attributes.finish_mark not exists for 2-th check, retry later [2024/05/06 19:15:11.403 +08:00] [INFO] [main.go:220] ["1 insert success: 2700"] [2024/05/06 19:15:11.410 +08:00] [INFO] [main.go:220] ["0 insert success: 2700"] [2024/05/06 19:15:11.572 +08:00] [INFO] [main.go:234] ["1 delete success: 1400"] [2024/05/06 19:15:11.573 +08:00] [INFO] [main.go:220] ["1 insert success: 2800"] [2024/05/06 19:15:11.578 +08:00] [INFO] [main.go:234] ["0 delete success: 1400"] [2024/05/06 19:15:11.579 +08:00] [INFO] [main.go:220] ["0 insert success: 2800"] [2024/05/06 19:15:11.749 +08:00] [INFO] [main.go:220] ["1 insert success: 2900"] [2024/05/06 19:15:11.750 +08:00] [INFO] [main.go:220] ["0 insert success: 2900"] [2024/05/06 19:15:11.907 +08:00] [INFO] [main.go:234] ["0 delete success: 1500"] [2024/05/06 19:15:11.908 +08:00] [INFO] [main.go:220] ["0 insert success: 3000"] [2024/05/06 19:15:11.910 +08:00] [INFO] [main.go:234] ["1 delete success: 1500"] [2024/05/06 19:15:11.911 +08:00] [INFO] [main.go:220] ["1 insert success: 3000"] table mark.finish_mark_3 not exists for 42-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:12 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/325983d5-feae-48b4-92de-4a1646500106 {"id":"325983d5-feae-48b4-92de-4a1646500106","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994109} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d191715 325983d5-feae-48b4-92de-4a1646500106 /tidb/cdc/default/default/upstream/7365843547324387948 {"id":7365843547324387948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/325983d5-feae-48b4-92de-4a1646500106 {"id":"325983d5-feae-48b4-92de-4a1646500106","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994109} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d191715 325983d5-feae-48b4-92de-4a1646500106 /tidb/cdc/default/default/upstream/7365843547324387948 {"id":7365843547324387948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/325983d5-feae-48b4-92de-4a1646500106 {"id":"325983d5-feae-48b4-92de-4a1646500106","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994109} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d191715 325983d5-feae-48b4-92de-4a1646500106 /tidb/cdc/default/default/upstream/7365843547324387948 {"id":7365843547324387948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5960.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-batch-add-table-test-8963?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 6ed3b850-c61b-4c91-a0b1-8a6ae9f1688e Info: {"upstream_id":7365843547324387948,"namespace":"default","id":"6ed3b850-c61b-4c91-a0b1-8a6ae9f1688e","sink_uri":"kafka://127.0.0.1:9092/ticdc-batch-add-table-test-8963?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:15:12.838074542+08:00","start_ts":449575416683560962,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575416683560962,"checkpoint_ts":449575416683560962,"checkpoint_time":"2024-05-06 19:15:12.715"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36757e100013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:13530, start at 2024-05-06 19:15:10.36713147 +0800 CST m=+5.771303329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:10.380 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:10.390 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:10.390 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367582c00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:13596, start at 2024-05-06 19:15:10.649600577 +0800 CST m=+5.996412797 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:10.659 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:10.640 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:10.640 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } split_and_random_merge scale: 40 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_attributes.finish_mark not exists for 3-th check, retry later + set +x [Mon May 6 19:15:14 CST 2024] <<<<<< START kafka consumer in batch_add_table case >>>>>> table mark.finish_mark_3 not exists for 43-th check, retry later table batch_add_table.finish_mark not exists for 1-th check, retry later [Mon May 6 19:15:14 CST 2024] <<<<<< START cdc server in kafka_simple_handle_key_only_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.1491214914.out server --log-file /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3675c7d00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:10695, start at 2024-05-06 19:15:15.071784773 +0800 CST m=+5.882385816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:15.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:15.060 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:15.060 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_attributes.finish_mark not exists for 4-th check, retry later table mark.finish_mark_3 not exists for 44-th check, retry later table batch_add_table.finish_mark not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/06 19:15:17.153 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/05/06 19:15:17.330 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/06 19:15:17.331 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/06 19:15:17.473 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/06 19:15:17.474 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/06 19:15:17.477 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/06 19:15:17.479 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3675c7d00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:10695, start at 2024-05-06 19:15:15.071784773 +0800 CST m=+5.882385816 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:15.082 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:15.060 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:15.060 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3675c8740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:10782, start at 2024-05-06 19:15:15.128612091 +0800 CST m=+5.866860872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:15.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:15.101 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:15.101 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh: line 1: 17047 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 (wd: /tmp/tidb_cdc_test/kafka_column_selector) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 [2024/05/06 19:15:17.624 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/06 19:15:17.634 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] table ddl_attributes.finish_mark not exists for 5-th check, retry later * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 [2024/05/06 19:15:17.834 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/06 19:15:17.835 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/06 19:15:17.847 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/06 19:15:17.849 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/06 19:15:18.045 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/06 19:15:18.058 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/06 19:15:18.252 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/06 19:15:18.254 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/06 19:15:18.275 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/06 19:15:18.277 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:18 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c504022-4132-495e-b321-39c941da266e {"id":"7c504022-4132-495e-b321-39c941da266e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994115} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d3711ca 7c504022-4132-495e-b321-39c941da266e /tidb/cdc/default/default/upstream/7365843578147234933 {"id":7365843578147234933,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c504022-4132-495e-b321-39c941da266e {"id":"7c504022-4132-495e-b321-39c941da266e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994115} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d3711ca 7c504022-4132-495e-b321-39c941da266e /tidb/cdc/default/default/upstream/7365843578147234933 {"id":7365843578147234933,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7c504022-4132-495e-b321-39c941da266e {"id":"7c504022-4132-495e-b321-39c941da266e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994115} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d3711ca 7c504022-4132-495e-b321-39c941da266e /tidb/cdc/default/default/upstream/7365843578147234933 {"id":7365843578147234933,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.14971.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/06 19:15:18.460 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/06 19:15:18.490 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/06 19:15:18.665 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/06 19:15:18.666 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/06 19:15:18.714 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/06 19:15:18.717 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table mark.finish_mark_3 not exists for 45-th check, retry later table batch_add_table.finish_mark exists check diff successfully [2024/05/06 19:15:18.921 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/06 19:15:18.973 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] wait process cdc.test exit for 1-th time... [2024/05/06 19:15:19.185 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/06 19:15:19.186 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/06 19:15:19.253 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/06 19:15:19.256 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/06 19:15:19.399 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/06 19:15:19.441 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] table ddl_attributes.finish_mark not exists for 6-th check, retry later [2024/05/06 19:15:19.636 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/06 19:15:19.637 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/06 19:15:19.686 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/06 19:15:19.689 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] wait process cdc.test exit for 2-th time... [2024/05/06 19:15:19.872 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/06 19:15:19.930 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12104.out cli tso query --pd=http://127.0.0.1:2379 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:15:20 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> + set +x + tso='449575418184335362 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575418184335362 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.15001.out cli changefeed create --start-ts=449575418184335362 '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-avro-16922?protocol=simple&encoding-format=avro' -c simple-handle-key-only-avro --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/conf/changefeed.toml *************************** 1. row *************************** count(distinct region_id): 3 [2024/05/06 19:15:20.175 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/06 19:15:20.177 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/06 19:15:20.240 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/06 19:15:20.244 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:19 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 49 100 24 100 25 357 372 --:--:-- --:--:-- --:--:-- 373 {"compatibility":"NONE"}The 1 times to try to start tidb cluster... Create changefeed successfully! ID: simple-handle-key-only-avro Info: {"upstream_id":7365843578147234933,"namespace":"default","id":"simple-handle-key-only-avro","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-avro-16922?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-06T19:15:20.461231802+08:00","start_ts":449575418184335362,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575418184335362,"checkpoint_ts":449575418184335362,"checkpoint_time":"2024-05-06 19:15:18.440"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:15:20.473 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/06 19:15:20.543 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/06 19:15:20.782 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/06 19:15:20.783 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] table mark.finish_mark_3 exists table mark.finish_mark exists check diff successfully [2024/05/06 19:15:20.860 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/06 19:15:20.864 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/06 19:15:21.097 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/06 19:15:21.187 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... [2024/05/06 19:15:21.443 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/06 19:15:21.444 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/06 19:15:21.533 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/06 19:15:21.542 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] table ddl_attributes.finish_mark exists check diff successfully [2024/05/06 19:15:21.836 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] + set +x + tso='449575418595901441 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575418595901441 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:15:21 CST 2024] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1215412156.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... [2024/05/06 19:15:21.953 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] wait process cdc.test exit for 1-th time... [2024/05/06 19:15:22.135 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/06 19:15:22.136 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] + set +x wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... [2024/05/06 19:15:21.697 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`base_for_view`] [where="((TRUE) AND TRUE)"] ["source checksum"=976018642] ["target checksum"=0] ["get source checksum cost"=2.023271ms] ["get target checksum cost"=1.790138ms] cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:15:22 CST 2024] <<<<<< run test case default_value success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:15:23 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> split_and_random_merge scale: 80 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:24 GMT < Content-Length: 859 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4dd1c96-1889-40fb-a50c-f399ee9ee44c {"id":"c4dd1c96-1889-40fb-a50c-f399ee9ee44c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994122} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d48e94c c4dd1c96-1889-40fb-a50c-f399ee9ee44c /tidb/cdc/default/default/upstream/7365843598190887897 {"id":7365843598190887897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4dd1c96-1889-40fb-a50c-f399ee9ee44c {"id":"c4dd1c96-1889-40fb-a50c-f399ee9ee44c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994122} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d48e94c c4dd1c96-1889-40fb-a50c-f399ee9ee44c /tidb/cdc/default/default/upstream/7365843598190887897 {"id":7365843598190887897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c4dd1c96-1889-40fb-a50c-f399ee9ee44c {"id":"c4dd1c96-1889-40fb-a50c-f399ee9ee44c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994122} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d48e94c c4dd1c96-1889-40fb-a50c-f399ee9ee44c /tidb/cdc/default/default/upstream/7365843598190887897 {"id":7365843598190887897,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12213.out cli changefeed create --start-ts=449575418595901441 '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-18167?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365843598190887897,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-18167?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:15:25.794172812+08:00","start_ts":449575418595901441,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575418595901441,"checkpoint_ts":449575418595901441,"checkpoint_time":"2024-05-06 19:15:20.010"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x [Mon May 6 19:15:27 CST 2024] <<<<<< START kafka consumer in cli_with_auth case >>>>>> table test.simple not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [2024/05/06 19:15:27.294 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m0.993063539s"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.15040.out cli changefeed pause -c simple-handle-key-only-avro ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_0 exists table mark.finish_mark_1 not exists for 1-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector_avro Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.15067.out cli changefeed update -c simple-handle-key-only-avro '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-avro-16922?protocol=simple&encoding-format=avro&max-message-bytes=650' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/conf/changefeed.toml --no-confirm table mark.finish_mark_1 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36769aa80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:10790, start at 2024-05-06 19:15:28.593549756 +0800 CST m=+5.296475870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:28.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:28.605 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:28.605 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/simple-handle-key-only-avro-16922?protocol=simple&encoding-format=avro To:kafka://127.0.0.1:9092/simple-handle-key-only-avro-16922?protocol=simple&encoding-format=avro&max-message-bytes=650} {Type:update Path:[Config SyncPointInterval] From: To:0xc0015f7d48} {Type:update Path:[Config SyncPointRetention] From: To:0xc0015f7d68} {Type:update Path:[Config Consistent] From: To:0xc000f9b0a0} Update changefeed config successfully! ID: simple-handle-key-only-avro Info: {"upstream_id":7365843578147234933,"namespace":"default","id":"simple-handle-key-only-avro","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-avro-16922?protocol=simple\u0026encoding-format=avro\u0026max-message-bytes=650","create_time":"2024-05-06T19:15:20.461231802+08:00","start_ts":449575418184335362,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":0,"checkpoint_ts":449575420347285506,"checkpoint_time":"2024-05-06 19:15:26.691"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release *************************** 1. row *************************** count(distinct region_id): 8 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.15091.out cli changefeed resume -c simple-handle-key-only-avro VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36769aa80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:10790, start at 2024-05-06 19:15:28.593549756 +0800 CST m=+5.296475870 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:28.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:28.605 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:28.605 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36769c28000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-c82s3-2nzrs, pid:10872, start at 2024-05-06 19:15:28.662430356 +0800 CST m=+5.309564209 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:28.669 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:28.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:28.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.simple exists table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575419015069726, "checkpoint_time": "2024-05-06 19:15:21.609", "error": null }' + echo '{ "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575419015069726, "checkpoint_time": "2024-05-06 19:15:21.609", "error": null }' { "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575419015069726, "checkpoint_time": "2024-05-06 19:15:21.609", "error": null } ++ echo '{' '"upstream_id":' 7365843598190887897, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575419015069726, '"checkpoint_time":' '"2024-05-06' '19:15:21.609",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365843598190887897, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575419015069726, '"checkpoint_time":' '"2024-05-06' '19:15:21.609",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 PASS table mark.finish_mark_1 not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 coverage: 2.1% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 [2024/05/06 19:15:32.808 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`base_for_view`] [where="((TRUE) AND TRUE)"] [cost=11.110091826s] [2024/05/06 19:15:32.810 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc002c20b00} {Type:update Path:[Config SyncPointRetention] From: To:0xc002c20b08} {Type:update Path:[Config Consistent] From: To:0xc0019781c0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12556.out cli changefeed --changefeed-id custom-changefeed-name pause + set +x table test.finish_mark not exists for 1-th check, retry later PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_1 not exists for 4-th check, retry later ***************** properties ***************** "readproportion"="0" "insertproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" "mysql.user"="root" "dotransactions"="false" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_auto_stop_1" "updateproportion"="0" "scanproportion"="0" "readallfields"="true" "threadcount"="4" "recordcount"="20" "operationcount"="0" "workload"="core" ********************************************** Run finished, takes 10.151438ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3420.5, Avg(us): 1922, Min(us): 1185, Max(us): 4249, 95th(us): 5000, 99th(us): 5000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "dotransactions"="false" "workload"="core" "recordcount"="20" "readproportion"="0" "threadcount"="4" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_auto_stop_2" "updateproportion"="0" "insertproportion"="0" "operationcount"="0" "mysql.port"="4000" "readallfields"="true" "scanproportion"="0" "mysql.user"="root" "requestdistribution"="uniform" ********************************************** Run finished, takes 13.091981ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2351.2, Avg(us): 2350, Min(us): 1047, Max(us): 4850, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "readproportion"="0" "readallfields"="true" "workload"="core" "operationcount"="0" "recordcount"="20" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "dotransactions"="false" "scanproportion"="0" "requestdistribution"="uniform" "threadcount"="4" "updateproportion"="0" "mysql.user"="root" "mysql.db"="changefeed_auto_stop_3" ********************************************** Run finished, takes 9.817812ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3644.8, Avg(us): 1862, Min(us): 1103, Max(us): 4241, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "mysql.db"="changefeed_auto_stop_4" "operationcount"="0" "updateproportion"="0" "scanproportion"="0" "mysql.port"="4000" "readallfields"="true" "mysql.host"="127.0.0.1" "readproportion"="0" "threadcount"="4" "recordcount"="20" "requestdistribution"="uniform" "dotransactions"="false" "mysql.user"="root" "workload"="core" "insertproportion"="0" ********************************************** Run finished, takes 10.33841ms INSERT - Takes(s): 0.0, Count: 19, OPS: 2993.2, Avg(us): 1810, Min(us): 1142, Max(us): 3850, 95th(us): 4000, 99th(us): 4000 [Mon May 6 19:15:34 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1247112473.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } + set +x table test.finish_mark not exists for 2-th check, retry later table mark.finish_mark_1 not exists for 5-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // timeout table region_merge.t1 exists check diff failed 1-th time, retry later [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } check diff successfully table test.finish_mark exists table mark.finish_mark_1 not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:37 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc361999-2257-47c3-92f5-906b5fe4bb4f {"id":"dc361999-2257-47c3-92f5-906b5fe4bb4f","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994135} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d802d36 dc361999-2257-47c3-92f5-906b5fe4bb4f /tidb/cdc/default/default/upstream/7365843659515057006 {"id":7365843659515057006,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc361999-2257-47c3-92f5-906b5fe4bb4f {"id":"dc361999-2257-47c3-92f5-906b5fe4bb4f","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994135} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d802d36 dc361999-2257-47c3-92f5-906b5fe4bb4f /tidb/cdc/default/default/upstream/7365843659515057006 {"id":7365843659515057006,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc361999-2257-47c3-92f5-906b5fe4bb4f {"id":"dc361999-2257-47c3-92f5-906b5fe4bb4f","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994135} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d802d36 dc361999-2257-47c3-92f5-906b5fe4bb4f /tidb/cdc/default/default/upstream/7365843659515057006 {"id":7365843659515057006,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:15:37 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1253112533.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36771e6c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:29733, start at 2024-05-06 19:15:36.992388273 +0800 CST m=+5.214106510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:36.999 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:36.987 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:36.987 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3677250c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:30628, start at 2024-05-06 19:15:37.452427873 +0800 CST m=+5.220416550 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:37.462 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:37.461 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:37.461 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575422069309442, "checkpoint_time": "2024-05-06 19:15:33.260", "error": null }' + echo '{ "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575422069309442, "checkpoint_time": "2024-05-06 19:15:33.260", "error": null }' { "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449575422069309442, "checkpoint_time": "2024-05-06 19:15:33.260", "error": null } ++ echo '{' '"upstream_id":' 7365843598190887897, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449575422069309442, '"checkpoint_time":' '"2024-05-06' '19:15:33.260",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365843598190887897, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449575422069309442, '"checkpoint_time":' '"2024-05-06' '19:15:33.260",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12639.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:15:39 CST 2024] <<<<<< run test case region_merge success! >>>>>> Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc004400c58} {Type:update Path:[Config SyncPointRetention] From: To:0xc004400c68} {Type:update Path:[Config Consistent] From: To:0xc0013860e0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} wait process cdc.test exit for 3-th time... Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365843598190887897,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-18167?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:15:25.794172812+08:00","start_ts":449575418595901441,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":0,"checkpoint_ts":449575422069309442,"checkpoint_time":"2024-05-06 19:15:33.260"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_1 not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36771e6c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:29733, start at 2024-05-06 19:15:36.992388273 +0800 CST m=+5.214106510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:36.999 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:36.987 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:36.987 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36771f240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:29802, start at 2024-05-06 19:15:37.062813066 +0800 CST m=+5.226378379 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:37.074 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:37.033 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:37.033 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:15:39 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3677250c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:30628, start at 2024-05-06 19:15:37.452427873 +0800 CST m=+5.220416550 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:37.462 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:37.461 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:37.461 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36772fe00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:30711, start at 2024-05-06 19:15:38.141183774 +0800 CST m=+5.859630541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:38.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:38.154 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:38.154 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12672.out cli changefeed --changefeed-id custom-changefeed-name resume PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:41 GMT < Content-Length: 1271 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc361999-2257-47c3-92f5-906b5fe4bb4f {"id":"dc361999-2257-47c3-92f5-906b5fe4bb4f","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994135} /tidb/cdc/default/__cdc_meta__/capture/eba509b7-4e2b-45b7-bd5f-41ad4a77c1cb {"id":"eba509b7-4e2b-45b7-bd5f-41ad4a77c1cb","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994138} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d802d36 dc361999-2257-47c3-92f5-906b5fe4bb4f /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d802d69 eba509b7-4e2b-45b7-bd5f-41ad4a77c1cb /tidb/cdc/default/default/upstream/7365843659515057006 {"id":7365843659515057006,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc361999-2257-47c3-92f5-906b5fe4bb4f {"id":"dc361999-2257-47c3-92f5-906b5fe4bb4f","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994135} /tidb/cdc/default/__cdc_meta__/capture/eba509b7-4e2b-45b7-bd5f-41ad4a77c1cb {"id":"eba509b7-4e2b-45b7-bd5f-41ad4a77c1cb","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994138} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d802d36 dc361999-2257-47c3-92f5-906b5fe4bb4f /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d802d69 eba509b7-4e2b-45b7-bd5f-41ad4a77c1cb /tidb/cdc/default/default/upstream/7365843659515057006 {"id":7365843659515057006,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc361999-2257-47c3-92f5-906b5fe4bb4f {"id":"dc361999-2257-47c3-92f5-906b5fe4bb4f","address":"127.0.0.1:8301","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994135} /tidb/cdc/default/__cdc_meta__/capture/eba509b7-4e2b-45b7-bd5f-41ad4a77c1cb {"id":"eba509b7-4e2b-45b7-bd5f-41ad4a77c1cb","address":"127.0.0.1:8302","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994138} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d802d36 dc361999-2257-47c3-92f5-906b5fe4bb4f /tidb/cdc/default/__cdc_meta__/owner/22318f4d9d802d69 eba509b7-4e2b-45b7-bd5f-41ad4a77c1cb /tidb/cdc/default/default/upstream/7365843659515057006 {"id":7365843659515057006,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Mon May 6 19:15:42 CST 2024] <<<<<< START kafka consumer in changefeed_auto_stop case >>>>>> check_changefeed_state http://127.0.0.1:2379 434fb15c-a005-4c9e-85a1-dd5e4032d0f3 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=434fb15c-a005-4c9e-85a1-dd5e4032d0f3 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 434fb15c-a005-4c9e-85a1-dd5e4032d0f3 -s [Mon May 6 19:15:42 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3114731149.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' table mark.finish_mark_1 not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7365843659515057006, "namespace": "default", "id": "434fb15c-a005-4c9e-85a1-dd5e4032d0f3", "state": "normal", "checkpoint_tso": 449575422185963522, "checkpoint_time": "2024-05-06 19:15:33.705", "error": null }' + echo '{ "upstream_id": 7365843659515057006, "namespace": "default", "id": "434fb15c-a005-4c9e-85a1-dd5e4032d0f3", "state": "normal", "checkpoint_tso": 449575422185963522, "checkpoint_time": "2024-05-06 19:15:33.705", "error": null }' { "upstream_id": 7365843659515057006, "namespace": "default", "id": "434fb15c-a005-4c9e-85a1-dd5e4032d0f3", "state": "normal", "checkpoint_tso": 449575422185963522, "checkpoint_time": "2024-05-06 19:15:33.705", "error": null } ++ echo '{' '"upstream_id":' 7365843659515057006, '"namespace":' '"default",' '"id":' '"434fb15c-a005-4c9e-85a1-dd5e4032d0f3",' '"state":' '"normal",' '"checkpoint_tso":' 449575422185963522, '"checkpoint_time":' '"2024-05-06' '19:15:33.705",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365843659515057006, '"namespace":' '"default",' '"id":' '"434fb15c-a005-4c9e-85a1-dd5e4032d0f3",' '"state":' '"normal",' '"checkpoint_tso":' 449575422185963522, '"checkpoint_time":' '"2024-05-06' '19:15:33.705",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Mon May 6 19:15:42 CST 2024] <<<<<< START cdc server in kafka_column_selector_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.3213332135.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x Starting Upstream TiDB... table mark.finish_mark_1 not exists for 9-th check, retry later Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_1.usertable not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:45 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8803ba66-2648-4881-a94a-84f56b649308 {"id":"8803ba66-2648-4881-a94a-84f56b649308","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994142} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9da14ece 8803ba66-2648-4881-a94a-84f56b649308 /tidb/cdc/default/default/upstream/7365843692695606308 {"id":7365843692695606308,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8803ba66-2648-4881-a94a-84f56b649308 {"id":"8803ba66-2648-4881-a94a-84f56b649308","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994142} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9da14ece 8803ba66-2648-4881-a94a-84f56b649308 /tidb/cdc/default/default/upstream/7365843692695606308 {"id":7365843692695606308,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8803ba66-2648-4881-a94a-84f56b649308 {"id":"8803ba66-2648-4881-a94a-84f56b649308","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994142} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9da14ece 8803ba66-2648-4881-a94a-84f56b649308 /tidb/cdc/default/default/upstream/7365843692695606308 {"id":7365843692695606308,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: b5cb4846-a9b9-4341-a70c-c1c0f518c410 Info: {"upstream_id":7365843692695606308,"namespace":"default","id":"b5cb4846-a9b9-4341-a70c-c1c0f518c410","sink_uri":"kafka://127.0.0.1:9092/ticdc-autorandom-test-16600?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:15:45.428159319+08:00","start_ts":449575425222115331,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575425222115331,"checkpoint_ts":449575425222115331,"checkpoint_time":"2024-05-06 19:15:45.287"} [Mon May 6 19:15:45 CST 2024] <<<<<< START kafka consumer in autorandom case >>>>>> table autorandom_test.table_a not exists for 1-th check, retry later table mark.finish_mark_1 not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:45 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54d74deb-640b-4e87-8a6a-8341b5948b65 {"id":"54d74deb-640b-4e87-8a6a-8341b5948b65","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994143} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9da100d0 54d74deb-640b-4e87-8a6a-8341b5948b65 /tidb/cdc/default/default/upstream/7365843689572563052 {"id":7365843689572563052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54d74deb-640b-4e87-8a6a-8341b5948b65 {"id":"54d74deb-640b-4e87-8a6a-8341b5948b65","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994143} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9da100d0 54d74deb-640b-4e87-8a6a-8341b5948b65 /tidb/cdc/default/default/upstream/7365843689572563052 {"id":7365843689572563052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54d74deb-640b-4e87-8a6a-8341b5948b65 {"id":"54d74deb-640b-4e87-8a6a-8341b5948b65","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994143} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9da100d0 54d74deb-640b-4e87-8a6a-8341b5948b65 /tidb/cdc/default/default/upstream/7365843689572563052 {"id":7365843689572563052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.cli.32197.out cli changefeed create --start-ts=449575424520617985 '--sink-uri=kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 Create changefeed successfully! ID: test Info: {"upstream_id":7365843689572563052,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true","create_time":"2024-05-06T19:15:46.199247819+08:00","start_ts":449575424520617985,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"column_selectors":[{"matcher":["test.*"],"columns":["*","!b"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575424520617985,"checkpoint_ts":449575424520617985,"checkpoint_time":"2024-05-06 19:15:42.611"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + info='{ "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575425215037444, "checkpoint_time": "2024-05-06 19:15:45.260", "error": null }' + echo '{ "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575425215037444, "checkpoint_time": "2024-05-06 19:15:45.260", "error": null }' { "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575425215037444, "checkpoint_time": "2024-05-06 19:15:45.260", "error": null } ++ echo '{' '"upstream_id":' 7365843598190887897, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575425215037444, '"checkpoint_time":' '"2024-05-06' '19:15:45.260",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365843598190887897, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575425215037444, '"checkpoint_time":' '"2024-05-06' '19:15:45.260",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12751.out cli changefeed --changefeed-id custom-changefeed-name remove table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449575425476919300 SinkURI: kafka://127.0.0.1:9092/ticdc-cli-test-18167?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 19:15:47 CST 2024] <<<<<< START kafka consumer in kafka_column_selector_avro case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml schema registry uri found: http://127.0.0.1:8088 table autorandom_test.table_a not exists for 2-th check, retry later table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting build checksum checker... table test.finishmark not exists for 1-th check, retry later table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table mark.finish_mark_2 not exists for 2-th check, retry later table test.finishmark not exists for 2-th check, retry later table autorandom_test.table_a exists check diff successfully table changefeed_auto_stop_4.usertable not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3677dea00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:20624, start at 2024-05-06 19:15:49.316990287 +0800 CST m=+5.381221829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:49.325 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:49.288 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:49.288 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:15:51 CST 2024] <<<<<< run test case autorandom success! >>>>>> changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12838.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-18167?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh: line 1: 15123 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_2 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3677dea00019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:20624, start at 2024-05-06 19:15:49.316990287 +0800 CST m=+5.381221829 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:49.325 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:49.288 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:49.288 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3677e0340017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:20698, start at 2024-05-06 19:15:49.428463944 +0800 CST m=+5.437895506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:49.437 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:49.438 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:49.438 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365843598190887897,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-18167?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:15:52.433667465+08:00","start_ts":449575427063152646,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575427063152646,"checkpoint_ts":449575427063152646,"checkpoint_time":"2024-05-06 19:15:52.310"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finishmark exists [2024/05/06 19:15:52.190 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/05/06 19:15:52.192 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/05/06 19:15:52.192 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=8.887392ms] [2024/05/06 19:15:52.199 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/05/06 19:15:52.202 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/05/06 19:15:52.202 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=9.562431ms] [2024/05/06 19:15:52.202 +08:00] [INFO] [main.go:95] ["compare checksum passed"] wait process cdc.test exit for 1-th time... table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:15:53 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_2 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.22159.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table mark.finish_mark_2 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... + set +x + tso='449575427660054529 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575427660054529 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_claim_check Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [Mon May 6 19:15:56 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.2221022212.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:15:56 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575428098359301, "checkpoint_time": "2024-05-06 19:15:56.259", "error": null }' + echo '{ "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575428098359301, "checkpoint_time": "2024-05-06 19:15:56.259", "error": null }' { "upstream_id": 7365843598190887897, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449575428098359301, "checkpoint_time": "2024-05-06 19:15:56.259", "error": null } ++ echo '{' '"upstream_id":' 7365843598190887897, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575428098359301, '"checkpoint_time":' '"2024-05-06' '19:15:56.259",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365843598190887897, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449575428098359301, '"checkpoint_time":' '"2024-05-06' '19:15:56.259",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] table mark.finish_mark_2 not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12969.out cli changefeed create --start-ts=449575418595901441 '--sink-uri=kafka://127.0.0.1:9093/ticdc-cli-test-ssl-12378?protocol=open-protocol&ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem&cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem&key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem&kafka-version=2.4.1&max-message-bytes=10485760&insecure-skip-verify=true' --tz=Asia/Shanghai [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: 9d6ab4de-6216-40b4-8eca-247aef154432 Info: {"upstream_id":7365843598190887897,"namespace":"default","id":"9d6ab4de-6216-40b4-8eca-247aef154432","sink_uri":"kafka://127.0.0.1:9093/ticdc-cli-test-ssl-12378?protocol=open-protocol\u0026ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem\u0026key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760\u0026insecure-skip-verify=true","create_time":"2024-05-06T19:15:59.00312578+08:00","start_ts":449575418595901441,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575418595901441,"checkpoint_ts":449575418595901441,"checkpoint_time":"2024-05-06 19:15:20.010"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... [2024/05/06 19:15:55.120 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.681221473s"] start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:15:59 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfa3617d-4936-4428-8db5-8f47ec100e6e {"id":"bfa3617d-4936-4428-8db5-8f47ec100e6e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994156} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9dccdef1 bfa3617d-4936-4428-8db5-8f47ec100e6e /tidb/cdc/default/default/upstream/7365843744142793694 {"id":7365843744142793694,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfa3617d-4936-4428-8db5-8f47ec100e6e {"id":"bfa3617d-4936-4428-8db5-8f47ec100e6e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994156} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9dccdef1 bfa3617d-4936-4428-8db5-8f47ec100e6e /tidb/cdc/default/default/upstream/7365843744142793694 {"id":7365843744142793694,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bfa3617d-4936-4428-8db5-8f47ec100e6e {"id":"bfa3617d-4936-4428-8db5-8f47ec100e6e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994156} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9dccdef1 bfa3617d-4936-4428-8db5-8f47ec100e6e /tidb/cdc/default/default/upstream/7365843744142793694 {"id":7365843744142793694,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.22262.out cli changefeed create --start-ts=449575427660054529 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-simple-test-5026?protocol=open-protocol&partition-num=4&kafka-client-id=cdc_test_simple&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 9f422276-c45c-4d1b-88a1-dda56de741df Info: {"upstream_id":7365843744142793694,"namespace":"default","id":"9f422276-c45c-4d1b-88a1-dda56de741df","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-simple-test-5026?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=cdc_test_simple\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:15:59.782277346+08:00","start_ts":449575427660054529,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575427660054529,"checkpoint_ts":449575427660054529,"checkpoint_time":"2024-05-06 19:15:54.587"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13017.out cli unsafe delete-service-gc-safepoint ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 19:16:01 CST 2024] <<<<<< START kafka consumer in simple case >>>>>> succeed to verify meta placement rules ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 2-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13050.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 3-th check, retry later + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3678bff40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7, pid:7139, start at 2024-05-06 19:16:03.75964433 +0800 CST m=+23.901424195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:03.765 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:03.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:03.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3678bff40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7, pid:7139, start at 2024-05-06 19:16:03.75964433 +0800 CST m=+23.901424195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:03.765 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:03.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:03.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3677bee80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-8v8v7, pid:7208, start at 2024-05-06 19:15:47.300369208 +0800 CST m=+7.388642913 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:17:47.308 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:15:47.308 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:05:47.308 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later table mark.finish_mark_3 not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Mon May 6 19:16:06 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.2007420076.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13159.out cli unsafe resolve-lock --region=128 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check data successfully table mark.finish_mark_3 not exists for 5-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... \033[0;36m<<< Run all test success >>>\033[0m =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: kafka... <<================= [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13192.out cli unsafe resolve-lock --region=128 --ts=449575430182404100 [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:16:09 CST 2024] <<<<<< run test case simple success! >>>>>> [Pipeline] // timeout [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:16:09 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2f5d022-f818-48ad-96d0-0b2dd4c9253c {"id":"b2f5d022-f818-48ad-96d0-0b2dd4c9253c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994166} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9dc0d8f6 b2f5d022-f818-48ad-96d0-0b2dd4c9253c /tidb/cdc/default/default/upstream/7365843730570820381 {"id":7365843730570820381,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2f5d022-f818-48ad-96d0-0b2dd4c9253c {"id":"b2f5d022-f818-48ad-96d0-0b2dd4c9253c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994166} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9dc0d8f6 b2f5d022-f818-48ad-96d0-0b2dd4c9253c /tidb/cdc/default/default/upstream/7365843730570820381 {"id":7365843730570820381,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2f5d022-f818-48ad-96d0-0b2dd4c9253c {"id":"b2f5d022-f818-48ad-96d0-0b2dd4c9253c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994166} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9dc0d8f6 b2f5d022-f818-48ad-96d0-0b2dd4c9253c /tidb/cdc/default/default/upstream/7365843730570820381 {"id":7365843730570820381,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.20397.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-rocks-test-30190?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' [Pipeline] // container [Pipeline] } PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] // withEnv [Pipeline] } Create changefeed successfully! ID: 223a4cff-c17b-4f7b-90ff-1fc1628a3145 Info: {"upstream_id":7365843730570820381,"namespace":"default","id":"223a4cff-c17b-4f7b-90ff-1fc1628a3145","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-rocks-test-30190?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:16:09.758337999+08:00","start_ts":449575431598243842,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575431598243842,"checkpoint_ts":449575431598243842,"checkpoint_time":"2024-05-06 19:16:09.610"} PASS [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } coverage: 2.4% of statements in github.com/pingcap/tiflow/... [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3678f8740012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:16209, start at 2024-05-06 19:16:07.351082716 +0800 CST m=+5.523156411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:07.361 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:07.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:07.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3678f8740012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:16209, start at 2024-05-06 19:16:07.351082716 +0800 CST m=+5.523156411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:07.361 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:07.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:07.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3678f93c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:16258, start at 2024-05-06 19:16:07.41104238 +0800 CST m=+5.514716328 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:07.421 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:07.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:07.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_3 exists table mark.finish_mark_4 not exists for 1-th check, retry later + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 120k 3490 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367922c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m, pid:7054, start at 2024-05-06 19:16:10.069276187 +0800 CST m=+5.099680925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:10.075 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:10.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:10.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367922c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m, pid:7054, start at 2024-05-06 19:16:10.069276187 +0800 CST m=+5.099680925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:10.075 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:10.032 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:10.032 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367925040005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-gdz5m, pid:7133, start at 2024-05-06 19:16:10.178146806 +0800 CST m=+5.158224937 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:10.187 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:10.177 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:10.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed + set +x [Mon May 6 19:16:11 CST 2024] <<<<<< START kafka consumer in multi_rocks case >>>>>> 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1415 100 678 100 737 1300 1413 --:--:-- --:--:-- --:--:-- 1411 100 1415 100 678 100 737 1300 1413 --:--:-- --:--:-- --:--:-- 1411 HTTP/1.1 201 Created Date: Mon, 06 May 2024 11:16:11 GMT Location: http://localhost:8083/connectors/my-connector Content-Type: application/json Content-Length: 678 Server: Jetty(9.4.51.v20230217) {"name":"my-connector","config":{"connector.class":"io.debezium.connector.mysql.MySqlConnector","tasks.max":"1","database.hostname":"127.0.0.1","database.port":"3310","database.user":"debezium","database.password":"dbz","database.server.id":"184054","topic.prefix":"dbserver1","schema.history.internal.kafka.bootstrap.servers":"127.0.0.1:9092","schema.history.internal.kafka.topic":"schemahistory.test","transforms":"x","transforms.x.type":"org.apache.kafka.connect.transforms.RegexRouter","transforms.x.regex":"(.*)","transforms.x.replacement":"output_debezium","binary.handling.mode":"base64","decimal.handling.mode":"double","name":"my-connector"},"tasks":[],"type":"source"}The 1 times to try to start tidb cluster... [Mon May 6 19:16:12 CST 2024] <<<<<< START cdc server in kafka_simple_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.1763117633.out server --log-file /tmp/tidb_cdc_test/kafka_simple_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_claim_check/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_4 not exists for 2-th check, retry later ***************** properties ***************** "table"="a1" "dotransactions"="false" "mysql.host"="127.0.0.1" "threadcount"="2" "operationcount"="0" "scanproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" "recordcount"="1000" "readproportion"="0" "readallfields"="true" "workload"="core" "mysql.db"="multi_rocks" "updateproportion"="0" "mysql.port"="4000" ********************************************** Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Run finished, takes 526.302111ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1948.7, Avg(us): 1024, Min(us): 814, Max(us): 13055, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.port"="4000" "table"="a2" "threadcount"="2" "insertproportion"="0" "dotransactions"="false" "operationcount"="0" "readproportion"="0" "mysql.user"="root" "readallfields"="true" "workload"="core" "mysql.host"="127.0.0.1" "mysql.db"="multi_rocks" "requestdistribution"="uniform" "updateproportion"="0" "recordcount"="1000" "scanproportion"="0" ********************************************** Run finished, takes 516.889581ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1998.5, Avg(us): 1006, Min(us): 769, Max(us): 16450, 95th(us): 2000, 99th(us): 2000 table mark.finish_mark_4 not exists for 3-th check, retry later } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 180k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v8.2.0-alpha-25-g58799adc6", "git_hash": "58799adc6338905a3d6177e5829f44667d1aa09e", "id": "9285d7c9-1ed1-425e-beee-f163938bfbfb", "pid": 12159, "is_owner": true Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) }wait process cdc.test exit for 1-th time... ***************** properties ***************** "readallfields"="true" "scanproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "readproportion"="0" "recordcount"="1000" "mysql.db"="multi_rocks" "insertproportion"="0" "mysql.port"="4000" "workload"="core" "table"="a3" "mysql.user"="root" "operationcount"="0" "requestdistribution"="uniform" "updateproportion"="0" "threadcount"="2" ********************************************** wait process cdc.test exit for 2-th time... Run finished, takes 528.638926ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1952.1, Avg(us): 1027, Min(us): 786, Max(us): 16305, 95th(us): 2000, 99th(us): 2000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:16:15 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e40276e-3736-4d93-8f78-cb4eaec2f22a {"id":"6e40276e-3736-4d93-8f78-cb4eaec2f22a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994172} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e1287d2 6e40276e-3736-4d93-8f78-cb4eaec2f22a /tidb/cdc/default/default/upstream/7365843821293377222 {"id":7365843821293377222,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e40276e-3736-4d93-8f78-cb4eaec2f22a {"id":"6e40276e-3736-4d93-8f78-cb4eaec2f22a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994172} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e1287d2 6e40276e-3736-4d93-8f78-cb4eaec2f22a /tidb/cdc/default/default/upstream/7365843821293377222 {"id":7365843821293377222,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6e40276e-3736-4d93-8f78-cb4eaec2f22a {"id":"6e40276e-3736-4d93-8f78-cb4eaec2f22a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994172} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e1287d2 6e40276e-3736-4d93-8f78-cb4eaec2f22a /tidb/cdc/default/default/upstream/7365843821293377222 {"id":7365843821293377222,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.17690.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 3-th time... ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.db"="multi_rocks" "recordcount"="1000" "mysql.user"="root" "table"="a4" "readproportion"="0" "workload"="core" "operationcount"="0" "scanproportion"="0" "dotransactions"="false" "insertproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "updateproportion"="0" "threadcount"="2" "mysql.port"="4000" ********************************************** table mark.finish_mark_4 not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:16:15 CST 2024] <<<<<< run test case cli_with_auth success! >>>>>> Run finished, takes 537.701992ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1918.6, Avg(us): 1044, Min(us): 762, Max(us): 16398, 95th(us): 2000, 99th(us): 2000 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.8612.out cli tso query --pd=http://127.0.0.1:2379 ***************** properties ***************** "recordcount"="1000" "mysql.port"="4000" "mysql.user"="root" "dotransactions"="false" "updateproportion"="0" "requestdistribution"="uniform" "table"="a5" "operationcount"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "threadcount"="2" "readallfields"="true" "readproportion"="0" "mysql.db"="multi_rocks" "workload"="core" "insertproportion"="0" ********************************************** ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449575433175040005 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575433175040005 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.17723.out cli changefeed create --start-ts=449575433175040005 '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-28587?protocol=simple' -c kafka-simple-claim-check --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/conf/changefeed.toml Run finished, takes 544.368474ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1891.3, Avg(us): 1056, Min(us): 731, Max(us): 15553, 95th(us): 2000, 99th(us): 2000 Create changefeed successfully! ID: kafka-simple-claim-check Info: {"upstream_id":7365843821293377222,"namespace":"default","id":"kafka-simple-claim-check","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-28587?protocol=simple","create_time":"2024-05-06T19:16:17.595630813+08:00","start_ts":449575433175040005,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575433175040005,"checkpoint_ts":449575433175040005,"checkpoint_time":"2024-05-06 19:16:15.625"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_4 not exists for 5-th check, retry later + set +x + tso='449575433294577668 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575433294577668 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:16:17 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.86508652.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table multi_rocks.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x start tidb cluster in /tmp/tidb_cdc_test/debezium Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table mark.finish_mark_4 not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3679b7640005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:32401, start at 2024-05-06 19:16:19.549622304 +0800 CST m=+5.213672618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:19.559 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:19.545 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:19.545 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:16:20 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd9d3ae8-b7ec-47d4-b458-0e75d4c671ba {"id":"bd9d3ae8-b7ec-47d4-b458-0e75d4c671ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994177} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e1f0fdb bd9d3ae8-b7ec-47d4-b458-0e75d4c671ba /tidb/cdc/default/default/upstream/7365843837613572792 {"id":7365843837613572792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd9d3ae8-b7ec-47d4-b458-0e75d4c671ba {"id":"bd9d3ae8-b7ec-47d4-b458-0e75d4c671ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994177} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e1f0fdb bd9d3ae8-b7ec-47d4-b458-0e75d4c671ba /tidb/cdc/default/default/upstream/7365843837613572792 {"id":7365843837613572792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd9d3ae8-b7ec-47d4-b458-0e75d4c671ba {"id":"bd9d3ae8-b7ec-47d4-b458-0e75d4c671ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994177} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e1f0fdb bd9d3ae8-b7ec-47d4-b458-0e75d4c671ba /tidb/cdc/default/default/upstream/7365843837613572792 {"id":7365843837613572792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 1d21cba0-2643-49dd-98cd-a7ce9151fb91 Info: {"upstream_id":7365843837613572792,"namespace":"default","id":"1d21cba0-2643-49dd-98cd-a7ce9151fb91","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-5193?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:16:20.848361486+08:00","start_ts":449575433294577668,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575433294577668,"checkpoint_ts":449575433294577668,"checkpoint_time":"2024-05-06 19:16:16.081"} [Mon May 6 19:16:20 CST 2024] <<<<<< START kafka consumer in common_1 case >>>>>> table multi_rocks.finish_mark not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_4 not exists for 7-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3679b7640005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:32401, start at 2024-05-06 19:16:19.549622304 +0800 CST m=+5.213672618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:19.559 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:19.545 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:19.545 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3679b9980013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-ds960-s5k2z, pid:32477, start at 2024-05-06 19:16:19.709444094 +0800 CST m=+5.318462354 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:19.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:19.686 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:19.686 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table multi_rocks.finish_mark not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table mark.finish_mark_4 not exists for 8-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_rocks.finish_mark exists check diff successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.33872.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 1-th time... + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.17773.out cli changefeed pause -c kafka-simple-claim-check [2024/05/06 19:16:22.222 +08:00] [WARN] [diff.go:182] ["table struct is not equal"] [reason="column num not equal, one is 5 another is 4"] wait process cdc.test exit for 2-th time... table mark.finish_mark_4 exists table mark.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:16:26 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> + set +x + tso='449575435631329281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575435631329281 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:16:26 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3391033912.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2864 0 --:--:-- --:--:-- --:--:-- 2870 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:16:14.228","puller_resolved_ts":"2024-05-06 19:16:07.227","last_synced_ts":"2024-05-06 19:13:58.427","now_ts":"2024-05-06 19:16:15.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:16:14.228","puller_resolved_ts":"2024-05-06' '19:16:07.227","last_synced_ts":"2024-05-06' '19:13:58.427","now_ts":"2024-05-06' '19:16:15.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9694 8.5 0.0 13382036 143968 ? Sl 19:13 0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9751 5.5 0.0 13561804 135944 ? Sl 19:13 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714994181.4324236,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00295bc00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994181.4325128,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994181.5011525,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002acbc00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714994181.501219,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994182.2495637,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00217c380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994182.2496316,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:16:26.129161+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00131e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:26.130594+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e1b180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:26.217691+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000deb180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.17804.out cli changefeed update -c kafka-simple-claim-check '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-28587?protocol=simple&max-message-bytes=2048' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/conf/changefeed.toml --no-confirm table mark.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/kafka-simple-claim-check-28587?protocol=simple To:kafka://127.0.0.1:9092/kafka-simple-claim-check-28587?protocol=simple&max-message-bytes=2048} {Type:update Path:[Config SyncPointInterval] From: To:0xc0018c3608} {Type:update Path:[Config SyncPointRetention] From: To:0xc0018c3618} {Type:update Path:[Config Consistent] From: To:0xc0012ba380} Update changefeed config successfully! ID: kafka-simple-claim-check Info: {"upstream_id":7365843821293377222,"namespace":"default","id":"kafka-simple-claim-check","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-28587?protocol=simple\u0026max-message-bytes=2048","create_time":"2024-05-06T19:16:17.595630813+08:00","start_ts":449575433175040005,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":0,"checkpoint_ts":449575435599872006,"checkpoint_time":"2024-05-06 19:16:24.875"} PASS =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:16:29 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6add279f-0bc4-46fe-b520-034d09820260 {"id":"6add279f-0bc4-46fe-b520-034d09820260","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994186} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e4598d7 6add279f-0bc4-46fe-b520-034d09820260 /tidb/cdc/default/default/upstream/7365843878256019821 {"id":7365843878256019821,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6add279f-0bc4-46fe-b520-034d09820260 {"id":"6add279f-0bc4-46fe-b520-034d09820260","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994186} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e4598d7 6add279f-0bc4-46fe-b520-034d09820260 /tidb/cdc/default/default/upstream/7365843878256019821 {"id":7365843878256019821,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6add279f-0bc4-46fe-b520-034d09820260 {"id":"6add279f-0bc4-46fe-b520-034d09820260","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994186} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e4598d7 6add279f-0bc4-46fe-b520-034d09820260 /tidb/cdc/default/default/upstream/7365843878256019821 {"id":7365843878256019821,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.33965.out cli changefeed create --start-ts=449575435631329281 '--sink-uri=kafka://127.0.0.1:9092/ticdc-generate-column-test-16753?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.17831.out cli changefeed resume -c kafka-simple-claim-check VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367a4a080014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:33396, start at 2024-05-06 19:16:28.948725879 +0800 CST m=+5.124142629 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:28.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:28.930 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:28.930 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367a4a080014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:33396, start at 2024-05-06 19:16:28.948725879 +0800 CST m=+5.124142629 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:28.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:28.930 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:28.930 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367a4b880015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-gs6jt-r3vbh, pid:33479, start at 2024-05-06 19:16:29.062830647 +0800 CST m=+5.189286966 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:29.070 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:29.026 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:29.026 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Create changefeed successfully! ID: c16e50bb-081f-45e0-b4a1-11c400b6d3cd Info: {"upstream_id":7365843878256019821,"namespace":"default","id":"c16e50bb-081f-45e0-b4a1-11c400b6d3cd","sink_uri":"kafka://127.0.0.1:9092/ticdc-generate-column-test-16753?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:16:29.998018652+08:00","start_ts":449575435631329281,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575435631329281,"checkpoint_ts":449575435631329281,"checkpoint_time":"2024-05-06 19:16:24.995"} PASS Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/debezium/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/debezium/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/debezium/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/debezium/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/debezium/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark not exists for 3-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x [Mon May 6 19:16:31 CST 2024] <<<<<< START kafka consumer in generate_column case >>>>>> table generate_column.t not exists for 1-th check, retry later + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table common_1.recover_and_insert not exists for 2-th check, retry later table mark.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 1-th check, retry later [Mon May 6 19:16:32 CST 2024] <<<<<< START cdc server in debezium case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.debezium.3491034912.out server --log-file /tmp/tidb_cdc_test/debezium/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/debezium/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 {"level":"warn","ts":"2024-05-06T19:16:32.130962+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00131e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:32.13191+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e1b180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:32.218472+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000deb180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later [2024/05/06 19:16:32.901 +08:00] [WARN] [diff.go:551] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] ["source checksum"=0] ["target checksum"=437042871] ["get source checksum cost"=1.849364ms] ["get target checksum cost"=1.29517ms] table mark.finish_mark not exists for 5-th check, retry later table generate_column.t not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:16:35 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/06555a74-4f18-4a27-bc98-eca5692302e2 {"id":"06555a74-4f18-4a27-bc98-eca5692302e2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994192} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e6e8dc3 06555a74-4f18-4a27-bc98-eca5692302e2 /tidb/cdc/default/default/upstream/7365843919071251778 {"id":7365843919071251778,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/06555a74-4f18-4a27-bc98-eca5692302e2 {"id":"06555a74-4f18-4a27-bc98-eca5692302e2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994192} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e6e8dc3 06555a74-4f18-4a27-bc98-eca5692302e2 /tidb/cdc/default/default/upstream/7365843919071251778 {"id":7365843919071251778,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/06555a74-4f18-4a27-bc98-eca5692302e2 {"id":"06555a74-4f18-4a27-bc98-eca5692302e2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994192} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e6e8dc3 06555a74-4f18-4a27-bc98-eca5692302e2 /tidb/cdc/default/default/upstream/7365843919071251778 {"id":7365843919071251778,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.debezium.cli.34971.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/output_ticdc?protocol=debezium&kafka-version=2.4.0' Create changefeed successfully! ID: 0094ffb3-5c8f-4020-b0b4-102940181625 Info: {"upstream_id":7365843919071251778,"namespace":"default","id":"0094ffb3-5c8f-4020-b0b4-102940181625","sink_uri":"kafka://127.0.0.1:9092/output_ticdc?protocol=debezium\u0026kafka-version=2.4.0","create_time":"2024-05-06T19:16:35.716350703+08:00","start_ts":449575438405861385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"debezium","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575438405861385,"checkpoint_ts":449575438405861385,"checkpoint_time":"2024-05-06 19:16:35.579"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table mark.finish_mark exists check diff successfully ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table generate_column.t exists table generate_column.t1 exists check diff failed 1-th time, retry later table common_1.finish_mark not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367aa9cc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:23592, start at 2024-05-06 19:16:35.060718472 +0800 CST m=+5.157880279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:35.069 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:35.059 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:35.059 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367aa9cc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:23592, start at 2024-05-06 19:16:35.060718472 +0800 CST m=+5.157880279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:35.069 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:35.059 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:35.059 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367aaa740012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:23673, start at 2024-05-06 19:16:35.118347796 +0800 CST m=+5.165487898 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:35.124 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:35.101 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:35.101 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... Starting Upstream TiDB... table test.finish_mark exists check diff successfully Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... + set +x go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20231116213047-1f7c1e02bcd4 go: downloading github.com/alecthomas/chroma v0.10.0 go: downloading go.uber.org/zap v1.26.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/google/uuid v1.3.1 go: downloading github.com/google/go-cmp v0.6.0 go: downloading github.com/segmentio/kafka-go v0.4.45 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20231117065153-a4f85c356873 go: downloading github.com/thessem/zap-prettyconsole v0.3.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/Code-Hex/dd v1.1.0 go: downloading github.com/klauspost/compress v1.17.1 go: downloading github.com/pierrec/lz4/v4 v4.1.15 go: downloading github.com/dlclark/regexp2 v1.4.0 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:16:37 CST 2024] <<<<<< run test case multi_source success! >>>>>> check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:16:38 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:38.131951+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00131e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:38.133286+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e1b180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:38.219569+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000deb180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table common_1.finish_mark not exists for 3-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.25009.out cli tso query --pd=http://127.0.0.1:2379 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) wait process cdc.test exit for 1-th time... [Pipeline] // cache [Pipeline] } [Pipeline] // dir ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container wait process cdc.test exit for 2-th time... [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:16:39 CST 2024] <<<<<< run test case generate_column success! >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 go: downloading github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 go: downloading golang.org/x/exp v0.0.0-20231006140011-7918f672742d go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/golang/protobuf v1.5.3 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20231114060955-8fc8a528217e go: downloading github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 go: downloading github.com/prometheus/client_model v0.5.0 go: downloading github.com/shirou/gopsutil/v3 v3.23.10 go: downloading github.com/prometheus/client_golang v1.17.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading google.golang.org/protobuf v1.31.0 go: downloading github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071 go: downloading google.golang.org/grpc v1.59.0 go: downloading github.com/cespare/xxhash/v2 v2.2.0 go: downloading github.com/prometheus/procfs v0.12.0 go: downloading github.com/prometheus/common v0.45.0 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/rogpeppe/go-internal v1.11.0 go: downloading github.com/tikv/pd/client v0.0.0-20231114041114-86831ce71865 go: downloading go.etcd.io/etcd/api/v3 v3.5.10 go: downloading go.etcd.io/etcd/client/v3 v3.5.10 go: downloading golang.org/x/sync v0.4.0 go: downloading github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.10 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b go: downloading google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b go: downloading golang.org/x/net v0.18.0 table common_1.finish_mark not exists for 4-th check, retry later + set +x + tso='449575439174205441 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575439174205441 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Mon May 6 19:16:40 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2505225054.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table common_1.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367b15700018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:14345, start at 2024-05-06 19:16:41.975653155 +0800 CST m=+5.245481858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:41.982 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:41.948 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:41.948 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:16:43 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0a3aa5d3-73eb-4e39-a039-a4e176e2e734 {"id":"0a3aa5d3-73eb-4e39-a039-a4e176e2e734","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994200} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e7e7edd 0a3aa5d3-73eb-4e39-a039-a4e176e2e734 /tidb/cdc/default/default/upstream/7365843941920631243 {"id":7365843941920631243,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0a3aa5d3-73eb-4e39-a039-a4e176e2e734 {"id":"0a3aa5d3-73eb-4e39-a039-a4e176e2e734","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994200} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e7e7edd 0a3aa5d3-73eb-4e39-a039-a4e176e2e734 /tidb/cdc/default/default/upstream/7365843941920631243 {"id":7365843941920631243,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0a3aa5d3-73eb-4e39-a039-a4e176e2e734 {"id":"0a3aa5d3-73eb-4e39-a039-a4e176e2e734","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994200} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e7e7edd 0a3aa5d3-73eb-4e39-a039-a4e176e2e734 /tidb/cdc/default/default/upstream/7365843941920631243 {"id":7365843941920631243,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:16:43 CST 2024] <<<<<< run test case common_1 success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:44.133006+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00131e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:44.134474+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e1b180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:44.22092+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000deb180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367b15700018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:14345, start at 2024-05-06 19:16:41.975653155 +0800 CST m=+5.245481858 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:41.982 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:41.948 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:41.948 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367b16100009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:14427, start at 2024-05-06 19:16:41.994980271 +0800 CST m=+5.213987772 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:18:42.003 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:16:41.988 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:06:41.988 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:46.120634+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e1b180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:16:46.120697+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:16:46.12135+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00131e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:16:46.121396+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:16:46.207922+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000deb180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:16:46.207967+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15768.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575441490771969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575441490771969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:16:48 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1580415806.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:50.13421+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00131e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:50.136268+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e1b180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh: line 1: 17866 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... {"level":"warn","ts":"2024-05-06T19:16:50.223015+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000deb180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:16:51 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/984955ff-8872-433c-af2c-270def8fce15 {"id":"984955ff-8872-433c-af2c-270def8fce15","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994209} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e9b26d4 984955ff-8872-433c-af2c-270def8fce15 /tidb/cdc/default/default/upstream/7365843975833238355 {"id":7365843975833238355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/984955ff-8872-433c-af2c-270def8fce15 {"id":"984955ff-8872-433c-af2c-270def8fce15","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994209} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e9b26d4 984955ff-8872-433c-af2c-270def8fce15 /tidb/cdc/default/default/upstream/7365843975833238355 {"id":7365843975833238355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/984955ff-8872-433c-af2c-270def8fce15 {"id":"984955ff-8872-433c-af2c-270def8fce15","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994209} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9e9b26d4 984955ff-8872-433c-af2c-270def8fce15 /tidb/cdc/default/default/upstream/7365843975833238355 {"id":7365843975833238355,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15859.out cli changefeed create --start-ts=449575441490771969 '--sink-uri=kafka://127.0.0.1:9092/ticdc-savepoint-test-7435?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 57e72f19-dd2a-4cb4-87dd-373d5eba489b Info: {"upstream_id":7365843975833238355,"namespace":"default","id":"57e72f19-dd2a-4cb4-87dd-373d5eba489b","sink_uri":"kafka://127.0.0.1:9092/ticdc-savepoint-test-7435?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:16:52.433170606+08:00","start_ts":449575441490771969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575441490771969,"checkpoint_ts":449575441490771969,"checkpoint_time":"2024-05-06 19:16:47.347"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_claim_check_avro Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x [Mon May 6 19:16:53 CST 2024] <<<<<< START kafka consumer in savepoint case >>>>>> table savepoint.finish_mark not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:16:56.135773+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00131e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:56.138118+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e1b180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:16:56.224034+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000deb180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1714994216.4331658,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00295bc00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994216.4332302,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994216.5020955,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002acbc00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714994216.5021358,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table savepoint.finish_mark not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1714994217.2514434,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00217c380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994217.2515097,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table savepoint.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:16:59 CST 2024] <<<<<< run test case savepoint success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:17:02.137674+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00131e000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:02.139505+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e1b180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:17:02.224462+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000deb180/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Mon May 6 19:17:03 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ true != \n\o ]] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2514025142.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367c81400003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:18911, start at 2024-05-06 19:17:05.233906437 +0800 CST m=+5.461524525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:05.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:05.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:05.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 7:17PM INF > Info cdc.mysql=kafka://127.0.0.1:9092/output_debezium cdc.tidb=kafka://127.0.0.1:9092/output_ticdc db.mysql=root@tcp(127.0.0.1:3310)/{db}?allowNativePasswords=true db.tidb=root@tcp(127.0.0.1:4000)/{db}?allowNativePasswords=true 7:17PM INF > Run case=sql/data_types.sql wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367c81400003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:18911, start at 2024-05-06 19:17:05.233906437 +0800 CST m=+5.461524525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:05.242 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:05.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:05.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367c82c00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:18973, start at 2024-05-06 19:17:05.332106918 +0800 CST m=+5.503338552 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:05.341 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:05.328 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:05.328 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 19:17:10 CST 2024] <<<<<< START cdc server in kafka_simple_claim_check_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.2039420396.out server --log-file /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:17:13 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d40ee009-acd3-492c-ae66-d59a8967a421 {"id":"d40ee009-acd3-492c-ae66-d59a8967a421","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994230} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9ef32ecc d40ee009-acd3-492c-ae66-d59a8967a421 /tidb/cdc/default/default/upstream/7365844073511134677 {"id":7365844073511134677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d40ee009-acd3-492c-ae66-d59a8967a421 {"id":"d40ee009-acd3-492c-ae66-d59a8967a421","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994230} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9ef32ecc d40ee009-acd3-492c-ae66-d59a8967a421 /tidb/cdc/default/default/upstream/7365844073511134677 {"id":7365844073511134677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d40ee009-acd3-492c-ae66-d59a8967a421 {"id":"d40ee009-acd3-492c-ae66-d59a8967a421","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994230} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9ef32ecc d40ee009-acd3-492c-ae66-d59a8967a421 /tidb/cdc/default/default/upstream/7365844073511134677 {"id":7365844073511134677,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.20439.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x + tso='449575448355274758 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575448355274758 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.20468.out cli changefeed create --start-ts=449575448355274758 '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-22106?protocol=simple&encoding-format=avro' -c kafka-simple-claim-check-avro --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/conf/changefeed.toml + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory Create changefeed successfully! ID: kafka-simple-claim-check-avro Info: {"upstream_id":7365844073511134677,"namespace":"default","id":"kafka-simple-claim-check-avro","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-22106?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-06T19:17:15.577097868+08:00","start_ts":449575448355274758,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-avro-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575448355274758,"checkpoint_ts":449575448355274758,"checkpoint_time":"2024-05-06 19:17:13.533"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.20513.out cli changefeed pause -c kafka-simple-claim-check-avro ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Mon May 6 19:17:23 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.20537.out cli changefeed update -c kafka-simple-claim-check-avro '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-22106?protocol=simple&encoding-format=avro&max-message-bytes=2048' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/conf/changefeed.toml --no-confirm VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367da67c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:17128, start at 2024-05-06 19:17:24.036943922 +0800 CST m=+5.152314507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:24.044 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:24.049 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:24.049 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367da67c0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:17128, start at 2024-05-06 19:17:24.036943922 +0800 CST m=+5.152314507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:24.044 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:24.049 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:24.049 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367da72c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:17220, start at 2024-05-06 19:17:24.078842528 +0800 CST m=+5.144812294 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:24.089 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:24.092 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:24.092 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-22106?protocol=simple&encoding-format=avro To:kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-22106?protocol=simple&encoding-format=avro&max-message-bytes=2048} {Type:update Path:[Config SyncPointInterval] From: To:0xc000e940b8} {Type:update Path:[Config SyncPointRetention] From: To:0xc000e940c8} {Type:update Path:[Config Consistent] From: To:0xc000e3a850} Update changefeed config successfully! ID: kafka-simple-claim-check-avro Info: {"upstream_id":7365844073511134677,"namespace":"default","id":"kafka-simple-claim-check-avro","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-22106?protocol=simple\u0026encoding-format=avro\u0026max-message-bytes=2048","create_time":"2024-05-06T19:17:15.577097868+08:00","start_ts":449575448355274758,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-avro-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":0,"checkpoint_ts":449575450793213956,"checkpoint_time":"2024-05-06 19:17:22.833"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.20569.out cli changefeed resume -c kafka-simple-claim-check-avro + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18542.out cli tso query --pd=http://127.0.0.1:2379 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449575452016377857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575452016377857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575452016377857 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 19:17:28 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1858218584.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x table test.finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367df94c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:13353, start at 2024-05-06 19:17:29.302826509 +0800 CST m=+5.475496928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:29.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:29.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:29.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367df94c0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:13353, start at 2024-05-06 19:17:29.302826509 +0800 CST m=+5.475496928 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:29.310 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:29.299 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:29.299 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367dfadc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:13425, start at 2024-05-06 19:17:29.419014679 +0800 CST m=+5.533754259 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:29.426 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:29.399 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:29.399 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:17:32 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52b3b96d-ec77-42ae-a369-12eb8ec7bd8c {"id":"52b3b96d-ec77-42ae-a369-12eb8ec7bd8c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994249} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f45aacf 52b3b96d-ec77-42ae-a369-12eb8ec7bd8c /tidb/cdc/default/default/upstream/7365844154385370102 {"id":7365844154385370102,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52b3b96d-ec77-42ae-a369-12eb8ec7bd8c {"id":"52b3b96d-ec77-42ae-a369-12eb8ec7bd8c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994249} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f45aacf 52b3b96d-ec77-42ae-a369-12eb8ec7bd8c /tidb/cdc/default/default/upstream/7365844154385370102 {"id":7365844154385370102,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52b3b96d-ec77-42ae-a369-12eb8ec7bd8c {"id":"52b3b96d-ec77-42ae-a369-12eb8ec7bd8c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994249} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f45aacf 52b3b96d-ec77-42ae-a369-12eb8ec7bd8c /tidb/cdc/default/default/upstream/7365844154385370102 {"id":7365844154385370102,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575452016377857 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18639.out cli changefeed create --start-ts=449575452016377857 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365844154385370102,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:17:32.486296901+08:00","start_ts":449575452016377857,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575452016377857,"checkpoint_ts":449575452016377857,"checkpoint_time":"2024-05-06 19:17:27.499"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2600 0 --:--:-- --:--:-- --:--:-- 2630 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:17:27.499","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:17:33.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:17:27.499","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:17:33.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:17:27.499","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:17:33.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-06 19:17:27.499' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:17:27.499","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:17:33.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:17:27.499","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:17:33.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-06 19:17:34' + echo 'sink_checkpoint_ts is 2024-05-06' 19:17:27.499 sink_checkpoint_ts is 2024-05-06 19:17:27.499 ++ date -d '2024-05-06 19:17:27.499' +%s + checkpoint_timestamp=1714994247 ++ date -d '2024-05-06 19:17:34' +%s + current_timestamp=1714994254 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' wait process cdc.test exit for 2-th time... + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14742.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:17:35 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x + tso='449575453838540801 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575453838540801 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575453838540801 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:17:35 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1477214774.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.t1 exists + sleep 5 start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:17:39 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c02e5b9-72be-4ec9-ab2f-08e1923726b2 {"id":"3c02e5b9-72be-4ec9-ab2f-08e1923726b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994256} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f52f3d2 3c02e5b9-72be-4ec9-ab2f-08e1923726b2 /tidb/cdc/default/default/upstream/7365844171608802195 {"id":7365844171608802195,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c02e5b9-72be-4ec9-ab2f-08e1923726b2 {"id":"3c02e5b9-72be-4ec9-ab2f-08e1923726b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994256} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f52f3d2 3c02e5b9-72be-4ec9-ab2f-08e1923726b2 /tidb/cdc/default/default/upstream/7365844171608802195 {"id":7365844171608802195,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3c02e5b9-72be-4ec9-ab2f-08e1923726b2 {"id":"3c02e5b9-72be-4ec9-ab2f-08e1923726b2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994256} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f52f3d2 3c02e5b9-72be-4ec9-ab2f-08e1923726b2 /tidb/cdc/default/default/upstream/7365844171608802195 {"id":7365844171608802195,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575453838540801 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14826.out cli changefeed create --start-ts=449575453838540801 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365844171608802195,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:17:39.460204489+08:00","start_ts":449575453838540801,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575453838540801,"checkpoint_ts":449575453838540801,"checkpoint_time":"2024-05-06 19:17:34.450"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 1615 0 --:--:-- --:--:-- --:--:-- 1609 100 243 100 243 0 0 1614 0 --:--:-- --:--:-- --:--:-- 1609 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:17:40.949","puller_resolved_ts":"2024-05-06 19:17:34.150","last_synced_ts":"2024-05-06 19:17:34.199","now_ts":"2024-05-06 19:17:41.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:40.949","puller_resolved_ts":"2024-05-06' '19:17:34.150","last_synced_ts":"2024-05-06' '19:17:34.199","now_ts":"2024-05-06' '19:17:41.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:40.949","puller_resolved_ts":"2024-05-06' '19:17:34.150","last_synced_ts":"2024-05-06' '19:17:34.199","now_ts":"2024-05-06' '19:17:41.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 table test.t1 exists + sleep 5 Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 12601 33.0 0.5 4724920 2275520 ? Sl 19:17 0:08 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 12602 23.6 0.5 4695732 2227540 ? Sl 19:17 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 12603 23.5 0.5 4694196 2223668 ? Sl 19:17 0:06 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 12605 31.9 0.5 4722356 2265092 ? Sl 19:17 0:08 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 5182 0 --:--:-- --:--:-- --:--:-- 5282 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:17:44.900","puller_resolved_ts":"2024-05-06 19:17:40.900","last_synced_ts":"2024-05-06 19:17:40.949","now_ts":"2024-05-06 19:17:48.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:44.900","puller_resolved_ts":"2024-05-06' '19:17:40.900","last_synced_ts":"2024-05-06' '19:17:40.949","now_ts":"2024-05-06' '19:17:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:44.900","puller_resolved_ts":"2024-05-06' '19:17:40.900","last_synced_ts":"2024-05-06' '19:17:40.949","now_ts":"2024-05-06' '19:17:48.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh: line 1: 20597 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367f26e40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:26251, start at 2024-05-06 19:17:48.621791583 +0800 CST m=+5.177442159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:48.629 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:48.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:48.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367f26e40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:26251, start at 2024-05-06 19:17:48.621791583 +0800 CST m=+5.177442159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:48.629 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:48.601 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:48.601 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367f28500013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:26324, start at 2024-05-06 19:17:48.714505894 +0800 CST m=+5.213401084 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:19:48.721 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:17:48.692 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:07:48.692 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/canal_json_adapter_compatibility Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... [Mon May 6 19:17:53 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.2770927711.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:17:56 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50932196-1824-4491-ab30-4db6c9f9c675 {"id":"50932196-1824-4491-ab30-4db6c9f9c675","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f9fb5cc 50932196-1824-4491-ab30-4db6c9f9c675 /tidb/cdc/default/default/upstream/7365844260573401513 {"id":7365844260573401513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50932196-1824-4491-ab30-4db6c9f9c675 {"id":"50932196-1824-4491-ab30-4db6c9f9c675","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f9fb5cc 50932196-1824-4491-ab30-4db6c9f9c675 /tidb/cdc/default/default/upstream/7365844260573401513 {"id":7365844260573401513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50932196-1824-4491-ab30-4db6c9f9c675 {"id":"50932196-1824-4491-ab30-4db6c9f9c675","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994274} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9f9fb5cc 50932196-1824-4491-ab30-4db6c9f9c675 /tidb/cdc/default/default/upstream/7365844260573401513 {"id":7365844260573401513,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.27769.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-event-filter-24595?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml Create changefeed successfully! ID: 52de8115-305f-4979-9d8b-623cd45aa1d5 Info: {"upstream_id":7365844260573401513,"namespace":"default","id":"52de8115-305f-4979-9d8b-623cd45aa1d5","sink_uri":"kafka://127.0.0.1:9092/ticdc-event-filter-24595?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:17:57.284853574+08:00","start_ts":449575459789471745,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575459789471745,"checkpoint_ts":449575459789471745,"checkpoint_time":"2024-05-06 19:17:57.151"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Mon May 6 19:17:58 CST 2024] <<<<<< START kafka consumer in event_filter case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t1 does not exists table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367ff5280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:21693, start at 2024-05-06 19:18:01.805020806 +0800 CST m=+5.405775202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:01.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:01.802 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:01.802 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367ff5280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:21693, start at 2024-05-06 19:18:01.805020806 +0800 CST m=+5.405775202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:01.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:01.802 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:01.802 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d367ff6f40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:21752, start at 2024-05-06 19:18:01.921124104 +0800 CST m=+5.461512055 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:01.930 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:01.917 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:01.917 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.23115.out cli tso query --pd=http://127.0.0.1:2379 table event_filter.t_alter exists + set +x + tso='449575462411173889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575462411173889 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:18:08 CST 2024] <<<<<< START cdc server in canal_json_adapter_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.2314123143.out server --log-file /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table event_filter.finish_mark exists check diff failed 1-th time, retry later [2024/05/06 19:18:07.668 +08:00] [WARN] [diff.go:745] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m34.767718506s] [2024/05/06 19:18:07.671 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:18:11 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb001ab6-ba71-499b-a45d-9bf4504b9baf {"id":"cb001ab6-ba71-499b-a45d-9bf4504b9baf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9fd829d6 cb001ab6-ba71-499b-a45d-9bf4504b9baf /tidb/cdc/default/default/upstream/7365844315630983686 {"id":7365844315630983686,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb001ab6-ba71-499b-a45d-9bf4504b9baf {"id":"cb001ab6-ba71-499b-a45d-9bf4504b9baf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9fd829d6 cb001ab6-ba71-499b-a45d-9bf4504b9baf /tidb/cdc/default/default/upstream/7365844315630983686 {"id":7365844315630983686,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb001ab6-ba71-499b-a45d-9bf4504b9baf {"id":"cb001ab6-ba71-499b-a45d-9bf4504b9baf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4d9fd829d6 cb001ab6-ba71-499b-a45d-9bf4504b9baf /tidb/cdc/default/default/upstream/7365844315630983686 {"id":7365844315630983686,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.23184.out cli changefeed create --start-ts=449575462411173889 '--sink-uri=kafka://127.0.0.1:9092/test?protocol=canal-json&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: b343e949-c905-4bcc-978f-7d445a70aa1f Info: {"upstream_id":7365844315630983686,"namespace":"default","id":"b343e949-c905-4bcc-978f-7d445a70aa1f","sink_uri":"kafka://127.0.0.1:9092/test?protocol=canal-json\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:18:12.17615866+08:00","start_ts":449575462411173889,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575462411173889,"checkpoint_ts":449575462411173889,"checkpoint_time":"2024-05-06 19:18:07.152"} PASS check diff failed 2-th time, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x check diff successfully wait process cdc.test exit for 1-th time... [2024/05/06 19:18:13.693 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:18:15 CST 2024] <<<<<< run test case event_filter success! >>>>>> wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:18:16 CST 2024] <<<<<< run test case cdc success! >>>>>> table test.binary_columns not exists for 1-th check, retry later table test.binary_columns not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.binary_columns not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table test.binary_columns not exists for 4-th check, retry later table test.binary_columns not exists for 5-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.binary_columns exists check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824b70001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29172, start at 2024-05-06 19:18:40.14020025 +0800 CST m=+5.057465287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824b70001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29172, start at 2024-05-06 19:18:40.14020025 +0800 CST m=+5.057465287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824ce80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29253, start at 2024-05-06 19:18:40.21867172 +0800 CST m=+5.084777059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36825bc4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1, pid:24742, start at 2024-05-06 19:18:41.148888561 +0800 CST m=+5.172110974 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:41.155 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:41.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:41.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.30592.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36825bc4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1, pid:24742, start at 2024-05-06 19:18:41.148888561 +0800 CST m=+5.172110974 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:41.155 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:41.137 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:41.137 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36825c740015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8m3lr-kk2c1, pid:24830, start at 2024-05-06 19:18:41.220505285 +0800 CST m=+5.184227746 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:41.226 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:41.231 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:41.231 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449575472003022849 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575472003022849 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575472003022849 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Mon May 6 19:18:45 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.3063330635.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.26279.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575472749084673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575472749084673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:18:48 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2632726329.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:18:48 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd763e5c-bc84-42f2-ae05-ed83ea404929 {"id":"dd763e5c-bc84-42f2-ae05-ed83ea404929","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994325} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da06b63d2 dd763e5c-bc84-42f2-ae05-ed83ea404929 /tidb/cdc/default/default/upstream/7365844481851719350 {"id":7365844481851719350,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd763e5c-bc84-42f2-ae05-ed83ea404929 {"id":"dd763e5c-bc84-42f2-ae05-ed83ea404929","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994325} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da06b63d2 dd763e5c-bc84-42f2-ae05-ed83ea404929 /tidb/cdc/default/default/upstream/7365844481851719350 {"id":7365844481851719350,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd763e5c-bc84-42f2-ae05-ed83ea404929 {"id":"dd763e5c-bc84-42f2-ae05-ed83ea404929","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994325} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da06b63d2 dd763e5c-bc84-42f2-ae05-ed83ea404929 /tidb/cdc/default/default/upstream/7365844481851719350 {"id":7365844481851719350,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575472003022849 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.30688.out cli changefeed create --start-ts=449575472003022849 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365844481851719350,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:18:48.725934837+08:00","start_ts":449575472003022849,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575472003022849,"checkpoint_ts":449575472003022849,"checkpoint_time":"2024-05-06 19:18:43.742"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 + '[' kafka == mysql ']' + stop_tidb_cluster + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:18:51 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fba465c-76b7-4aef-9fa3-1ec282ff105f {"id":"5fba465c-76b7-4aef-9fa3-1ec282ff105f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994328} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da06ee3fa 5fba465c-76b7-4aef-9fa3-1ec282ff105f /tidb/cdc/default/default/upstream/7365844482888493054 {"id":7365844482888493054,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fba465c-76b7-4aef-9fa3-1ec282ff105f {"id":"5fba465c-76b7-4aef-9fa3-1ec282ff105f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994328} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da06ee3fa 5fba465c-76b7-4aef-9fa3-1ec282ff105f /tidb/cdc/default/default/upstream/7365844482888493054 {"id":7365844482888493054,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5fba465c-76b7-4aef-9fa3-1ec282ff105f {"id":"5fba465c-76b7-4aef-9fa3-1ec282ff105f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994328} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da06ee3fa 5fba465c-76b7-4aef-9fa3-1ec282ff105f /tidb/cdc/default/default/upstream/7365844482888493054 {"id":7365844482888493054,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.26379.out cli changefeed create --start-ts=449575472749084673 '--sink-uri=kafka://127.0.0.1:9092/ticdc-split-region-test-24446?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml Create changefeed successfully! ID: 02f381fb-b92c-4489-89e1-083df15d254f Info: {"upstream_id":7365844482888493054,"namespace":"default","id":"02f381fb-b92c-4489-89e1-083df15d254f","sink_uri":"kafka://127.0.0.1:9092/ticdc-split-region-test-24446?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:18:51.894137383+08:00","start_ts":449575472749084673,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575472749084673,"checkpoint_ts":449575472749084673,"checkpoint_time":"2024-05-06 19:18:46.588"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.binary_columns exists check diff failed 1-th time, retry later + set +x [Mon May 6 19:18:53 CST 2024] <<<<<< START kafka consumer in split_region case >>>>>> table split_region.test1 not exists for 1-th check, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later table split_region.test1 not exists for 2-th check, retry later 7:18PM INF > Run case=sql/debezium/binary_column_test.sql table split_region.test1 exists table split_region.test2 exists check diff failed 1-th time, retry later check diff failed 4-th time, retry later check diff successfully check diff failed 5-th time, retry later 7:19PM INF > Run case=sql/debezium/binary_mode_test.sql check diff failed 6-th time, retry later + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:19:05 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> 7:19PM INF > Run case=sql/debezium/connector_test.sql Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824b70001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29172, start at 2024-05-06 19:18:40.14020025 +0800 CST m=+5.057465287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824b70001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29172, start at 2024-05-06 19:18:40.14020025 +0800 CST m=+5.057465287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824ce80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29253, start at 2024-05-06 19:18:40.21867172 +0800 CST m=+5.084777059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:19:17 CST 2024] <<<<<< run test case split_region success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/canal_json_basic Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36853a6c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:24483, start at 2024-05-06 19:19:28.201822914 +0800 CST m=+5.300467897 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:21:28.209 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:19:28.205 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:09:28.205 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36853a6c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:24483, start at 2024-05-06 19:19:28.201822914 +0800 CST m=+5.300467897 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:21:28.209 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:19:28.205 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:09:28.205 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36853b140015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:24551, start at 2024-05-06 19:19:28.239990919 +0800 CST m=+5.277839230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:21:28.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:19:28.247 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:09:28.247 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Mon May 6 19:19:33 CST 2024] <<<<<< START cdc server in canal_json_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.2598325985.out server --log-file /tmp/tidb_cdc_test/canal_json_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_basic/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824b70001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29172, start at 2024-05-06 19:18:40.14020025 +0800 CST m=+5.057465287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824b70001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29172, start at 2024-05-06 19:18:40.14020025 +0800 CST m=+5.057465287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824ce80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29253, start at 2024-05-06 19:18:40.21867172 +0800 CST m=+5.084777059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:19:36 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78307699-0ac2-4e38-9067-47deab503c7d {"id":"78307699-0ac2-4e38-9067-47deab503c7d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994373} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da12a0fcb 78307699-0ac2-4e38-9067-47deab503c7d /tidb/cdc/default/default/upstream/7365844685492913169 {"id":7365844685492913169,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78307699-0ac2-4e38-9067-47deab503c7d {"id":"78307699-0ac2-4e38-9067-47deab503c7d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994373} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da12a0fcb 78307699-0ac2-4e38-9067-47deab503c7d /tidb/cdc/default/default/upstream/7365844685492913169 {"id":7365844685492913169,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/78307699-0ac2-4e38-9067-47deab503c7d {"id":"78307699-0ac2-4e38-9067-47deab503c7d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994373} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da12a0fcb 78307699-0ac2-4e38-9067-47deab503c7d /tidb/cdc/default/default/upstream/7365844685492913169 {"id":7365844685492913169,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.cli.26028.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-canal-json-basic?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 6b909e1e-787c-4640-a524-27b9f531b636 Info: {"upstream_id":7365844685492913169,"namespace":"default","id":"6b909e1e-787c-4640-a524-27b9f531b636","sink_uri":"kafka://127.0.0.1:9092/ticdc-canal-json-basic?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-06T19:19:36.812822612+08:00","start_ts":449575485886955523,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575485886955523,"checkpoint_ts":449575485886955523,"checkpoint_time":"2024-05-06 19:19:36.705"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... 7:19PM INF > Run case=sql/debezium/connector_test_ro.sql + set +x [Mon May 6 19:19:43 CST 2024] <<<<<< START kafka consumer in canal_json_basic case >>>>>> start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824b70001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29172, start at 2024-05-06 19:18:40.14020025 +0800 CST m=+5.057465287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824b70001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29172, start at 2024-05-06 19:18:40.14020025 +0800 CST m=+5.057465287 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.148 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36824ce80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8bdr9-7xpj9, pid:29253, start at 2024-05-06 19:18:40.21867172 +0800 CST m=+5.084777059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:20:40.224 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:18:40.186 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:08:40.186 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.34795.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark not exists for 5-th check, retry later + set +x + tso='449575490787737601 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575490787737601 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575490787737601 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Mon May 6 19:19:56 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.3482734829.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:19:59 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365844481851719350 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:18:48.725934837 +0800 CST StartTs:449575472003022849 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00267a1b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575473300635654} {CheckpointTs:449575473680744451 MinTableBarrierTs:449575473680744451 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/503075ab-6f5e-4547-a384-36e19c659963 {"id":"503075ab-6f5e-4547-a384-36e19c659963","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994397} /tidb/cdc/default/__cdc_meta__/capture/dd763e5c-bc84-42f2-ae05-ed83ea404929 {"id":"dd763e5c-bc84-42f2-ae05-ed83ea404929","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994325} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da06b63d2 dd763e5c-bc84-42f2-ae05-ed83ea404929 /tidb/cdc/default/__cdc_meta__/owner/22318f4da18da4d2 503075ab-6f5e-4547-a384-36e19c659963 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365844481851719350,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:18:48.725934837+08:00","start-ts":449575472003022849,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575473300635654} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449575473680744451,"min-table-barrier-ts":449575473680744451,"admin-job-type":0} /tidb/cdc/default/default/task/position/503075ab-6f5e-4547-a384-36e19c659963/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dd763e5c-bc84-42f2-ae05-ed83ea404929/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365844481851719350 {"id":7365844481851719350,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365844481851719350 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:18:48.725934837 +0800 CST StartTs:449575472003022849 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00267a1b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575473300635654} {CheckpointTs:449575473680744451 MinTableBarrierTs:449575473680744451 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/503075ab-6f5e-4547-a384-36e19c659963 {"id":"503075ab-6f5e-4547-a384-36e19c659963","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994397} /tidb/cdc/default/__cdc_meta__/capture/dd763e5c-bc84-42f2-ae05-ed83ea404929 {"id":"dd763e5c-bc84-42f2-ae05-ed83ea404929","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994325} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da06b63d2 dd763e5c-bc84-42f2-ae05-ed83ea404929 /tidb/cdc/default/__cdc_meta__/owner/22318f4da18da4d2 503075ab-6f5e-4547-a384-36e19c659963 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365844481851719350,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:18:48.725934837+08:00","start-ts":449575472003022849,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575473300635654} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449575473680744451,"min-table-barrier-ts":449575473680744451,"admin-job-type":0} /tidb/cdc/default/default/task/position/503075ab-6f5e-4547-a384-36e19c659963/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dd763e5c-bc84-42f2-ae05-ed83ea404929/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/upstream/7365844481851719350 {"id":7365844481851719350,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365844481851719350 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-06 19:18:48.725934837 +0800 CST StartTs:449575472003022849 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00267a1b0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-25-g58799adc6 Epoch:449575473300635654} {CheckpointTs:449575473680744451 MinTableBarrierTs:449575473680744451 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/503075ab-6f5e-4547-a384-36e19c659963 {"id":"503075ab-6f5e-4547-a384-36e19c659963","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994397} /tidb/cdc/default/__cdc_meta__/capture/dd763e5c-bc84-42f2-ae05-ed83ea404929 {"id":"dd763e5c-bc84-42f2-ae05-ed83ea404929","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994325} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da06b63d2 dd763e5c-bc84-42f2-ae05-ed83ea404929 /tidb/cdc/default/__cdc_meta__/owner/22318f4da18da4d2 503075ab-6f5e-4547-a384-36e19c659963 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365844481851719350,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-06T19:18:48.725934837+08:00","start-ts":449575472003022849,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-25-g58799adc6","epoch":449575473300635654} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449575473680744451,"min-table-barrier-ts":449575473680744451,"admin-job-type":0} /tidb/cdc/default/default/task/position/503075ab-6f5e-4547-a384-36e19c659963/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dd763e5c-bc84-42f2-ae05-ed83ea404929/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365844481851719350 {"id":7365844481851719350,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575490787737601 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.34880.out cli changefeed create --start-ts=449575490787737601 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 table test.finish_mark not exists for 7-th check, retry later table test.finish_mark not exists for 8-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2621 0 --:--:-- --:--:-- --:--:-- 2630 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:19:51.049","puller_resolved_ts":"2024-05-06 19:19:43.049","last_synced_ts":"2024-05-06 19:17:34.199","now_ts":"2024-05-06 19:19:51.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:19:51.049","puller_resolved_ts":"2024-05-06' '19:19:43.049","last_synced_ts":"2024-05-06' '19:17:34.199","now_ts":"2024-05-06' '19:19:51.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 16235 7.8 0.0 13521944 143004 ? Sl 19:17 0:12 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 16304 5.0 0.0 13325272 137392 ? Sl 19:17 0:07 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714994399.0755856,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0005e5c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994399.0756443,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994399.1170104,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020a9500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994399.1170516,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994399.1195807,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003d96700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714994399.119617,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} Create changefeed successfully! ID: test-2 Info: {"upstream_id":7365844481851719350,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:20:02.90818869+08:00","start_ts":449575490787737601,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575490787737601,"checkpoint_ts":449575490787737601,"checkpoint_time":"2024-05-06 19:19:55.400"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... {"level":"warn","ts":"2024-05-06T19:20:03.945004+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e67180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:03.945272+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012081c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:03.991453+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012c4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark not exists for 9-th check, retry later + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 table test.finish_mark not exists for 10-th check, retry later table test.finish_mark not exists for 11-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 11432 0 --:--:-- --:--:-- --:--:-- 11476 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:17:47.900","puller_resolved_ts":"2024-05-06 19:17:47.900","last_synced_ts":"2024-05-06 19:17:40.949","now_ts":"2024-05-06 19:19:58.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:47.900","puller_resolved_ts":"2024-05-06' '19:17:47.900","last_synced_ts":"2024-05-06' '19:17:40.949","now_ts":"2024-05-06' '19:19:58.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:17:47.900","puller_resolved_ts":"2024-05-06' '19:17:47.900","last_synced_ts":"2024-05-06' '19:17:40.949","now_ts":"2024-05-06' '19:19:58.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory {"level":"warn","ts":"2024-05-06T19:20:09.946169+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012081c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:09.946779+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e67180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:09.992482+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012c4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark not exists for 12-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table test.finish_mark not exists for 13-th check, retry later table test.finish_mark not exists for 14-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:20:15.947548+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012081c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:15.948715+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e67180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:15.992936+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012c4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark not exists for 15-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 16-th check, retry later table test.finish_mark not exists for 17-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:20:21.94916+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012081c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:21.94975+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e67180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:21.994506+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012c4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:20:23.935499+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012081c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:20:23.935543+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:20:23.936173+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e67180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:20:23.936219+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-06T19:20:23.984027+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012c4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-06T19:20:23.984075+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 7:20PM INF > Run case=sql/debezium/datetime_key_test.sql 7:20PM INF > Run case=sql/debezium/db_default_charset.sql VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368882bc0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:16080, start at 2024-05-06 19:20:21.943515133 +0800 CST m=+5.265439126 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:22:21.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:20:21.935 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:10:21.935 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368882bc0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:16080, start at 2024-05-06 19:20:21.943515133 +0800 CST m=+5.265439126 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:22:21.952 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:20:21.935 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:10:21.935 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3688835c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:16157, start at 2024-05-06 19:20:22.022441503 +0800 CST m=+5.296600131 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:22:22.029 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:20:22.026 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:10:22.026 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.17488.out cli tso query --pd=http://127.0.0.1:2379 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:20:27.950285+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012081c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:27.951004+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e67180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 7:20PM INF > Run case=sql/debezium/db_default_charset_noutf.sql {"level":"warn","ts":"2024-05-06T19:20:27.996316+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012c4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark not exists for 1-th check, retry later + set +x + tso='449575499172413441 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575499172413441 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575499172413441 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:20:28 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1752017522.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:20:31 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6baf9de8-32d6-4ee3-a473-3e54c708c14e {"id":"6baf9de8-32d6-4ee3-a473-3e54c708c14e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994429} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da1f83dd4 6baf9de8-32d6-4ee3-a473-3e54c708c14e /tidb/cdc/default/default/upstream/7365844920127261989 {"id":7365844920127261989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6baf9de8-32d6-4ee3-a473-3e54c708c14e {"id":"6baf9de8-32d6-4ee3-a473-3e54c708c14e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994429} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da1f83dd4 6baf9de8-32d6-4ee3-a473-3e54c708c14e /tidb/cdc/default/default/upstream/7365844920127261989 {"id":7365844920127261989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6baf9de8-32d6-4ee3-a473-3e54c708c14e {"id":"6baf9de8-32d6-4ee3-a473-3e54c708c14e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994429} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da1f83dd4 6baf9de8-32d6-4ee3-a473-3e54c708c14e /tidb/cdc/default/default/upstream/7365844920127261989 {"id":7365844920127261989,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575499172413441 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.17577.out cli changefeed create --start-ts=449575499172413441 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365844920127261989,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:20:32.350482512+08:00","start_ts":449575499172413441,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575499172413441,"checkpoint_ts":449575499172413441,"checkpoint_time":"2024-05-06 19:20:27.385"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... 7:20PM INF > Run case=sql/debezium/decimal_column_test.sql table test.finish_mark not exists for 4-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:20:33.951487+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012081c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:33.952683+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e67180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:33.997481+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012c4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' {"level":"warn","ts":1714994434.0771115,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0005e5c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994434.0771458,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994434.11851,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020a9500/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714994434.1185365,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714994434.1210296,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003d96700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714994434.1210654,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.finish_mark not exists for 5-th check, retry later table test.t1 exists + sleep 5 table test.finish_mark not exists for 6-th check, retry later 7:20PM INF > Run case=sql/debezium/enum_column_test.sql table test.finish_mark not exists for 7-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0{"level":"warn","ts":"2024-05-06T19:20:39.952497+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012081c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:39.954224+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e67180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-06T19:20:39.99848+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0012c4000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test table test.finish_mark not exists for 8-th check, retry later + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 16076 4.0 0.0 2284972 198400 ? Sl 19:20 0:01 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714994416670086697.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb.log jenkins 16080 13.4 0.0 2779584 260636 ? Sl 19:20 0:03 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714994416672848942.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_other.log jenkins 16157 13.9 0.0 2649940 284296 ? Sl 19:20 0:03 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714994416719456838.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2302 0 --:--:-- --:--:-- --:--:-- 2314 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:20:39.735","puller_resolved_ts":"2024-05-06 19:20:33.835","last_synced_ts":"2024-05-06 19:20:34.285","now_ts":"2024-05-06 19:20:41.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:20:39.735","puller_resolved_ts":"2024-05-06' '19:20:33.835","last_synced_ts":"2024-05-06' '19:20:34.285","now_ts":"2024-05-06' '19:20:41.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:20:39.735","puller_resolved_ts":"2024-05-06' '19:20:33.835","last_synced_ts":"2024-05-06' '19:20:34.285","now_ts":"2024-05-06' '19:20:41.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:20:45 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> 7:20PM INF > Run case=sql/debezium/multitable_dbz_871.sql + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory 7:20PM INF > Run case=sql/debezium/mysql_dbz_6533.sql chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + '[' kafka == mysql ']' + stop_tidb_cluster ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/canal_json_content_compatible Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 7:21PM INF > Run case=sql/debezium/nationalized_character_test.sql + stop_tidb_cluster VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368b19ec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:19992, start at 2024-05-06 19:21:04.42488817 +0800 CST m=+5.130277893 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:23:04.433 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:21:04.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:11:04.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368b19ec0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:19992, start at 2024-05-06 19:21:04.42488817 +0800 CST m=+5.130277893 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:23:04.433 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:21:04.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:11:04.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368b1a9c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:20091, start at 2024-05-06 19:21:04.423946717 +0800 CST m=+5.078654246 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:23:04.431 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:21:04.423 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:11:04.423 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21406.out cli tso query --pd=http://127.0.0.1:2379 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449575509787672577 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575509787672577 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575509787672577 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 19:21:09 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2144621448.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 7:21PM INF > Run case=sql/debezium/numeric_column_test.sql ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:21:12 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6055a27c-0656-4551-9f26-123ee27a84de {"id":"6055a27c-0656-4551-9f26-123ee27a84de","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da29cabd2 6055a27c-0656-4551-9f26-123ee27a84de /tidb/cdc/default/default/upstream/7365845093549686368 {"id":7365845093549686368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6055a27c-0656-4551-9f26-123ee27a84de {"id":"6055a27c-0656-4551-9f26-123ee27a84de","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da29cabd2 6055a27c-0656-4551-9f26-123ee27a84de /tidb/cdc/default/default/upstream/7365845093549686368 {"id":7365845093549686368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6055a27c-0656-4551-9f26-123ee27a84de {"id":"6055a27c-0656-4551-9f26-123ee27a84de","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994469} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da29cabd2 6055a27c-0656-4551-9f26-123ee27a84de /tidb/cdc/default/default/upstream/7365845093549686368 {"id":7365845093549686368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575509787672577 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21504.out cli changefeed create --start-ts=449575509787672577 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365845093549686368,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:21:12.883163608+08:00","start_ts":449575509787672577,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575509787672577,"checkpoint_ts":449575509787672577,"checkpoint_time":"2024-05-06 19:21:07.879"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368b8ef80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:27396, start at 2024-05-06 19:21:11.907788685 +0800 CST m=+5.388976094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:23:11.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:21:11.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:11:11.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368b8ef80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:27396, start at 2024-05-06 19:21:11.907788685 +0800 CST m=+5.388976094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:23:11.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:21:11.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:11:11.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d368b90680014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:27468, start at 2024-05-06 19:21:12.000950696 +0800 CST m=+5.417034513 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:23:12.010 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:21:12.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:11:12.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } 7:21PM INF > Run case=sql/debezium/readbinlog_test.sql table test.t1 exists + sleep 5 [Mon May 6 19:21:16 CST 2024] <<<<<< START cdc server in canal_json_content_compatible case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_content_compatible.2879028792.out server --log-file /tmp/tidb_cdc_test/canal_json_content_compatible/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_content_compatible/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 7:21PM INF > Run case=sql/debezium/real_test.sql + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:21:19 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b46bdf47-fec6-4cb8-adf7-b8e8c3a06c13 {"id":"b46bdf47-fec6-4cb8-adf7-b8e8c3a06c13","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994477} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da2bca9ce b46bdf47-fec6-4cb8-adf7-b8e8c3a06c13 /tidb/cdc/default/default/upstream/7365845127484498177 {"id":7365845127484498177,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b46bdf47-fec6-4cb8-adf7-b8e8c3a06c13 {"id":"b46bdf47-fec6-4cb8-adf7-b8e8c3a06c13","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994477} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da2bca9ce b46bdf47-fec6-4cb8-adf7-b8e8c3a06c13 /tidb/cdc/default/default/upstream/7365845127484498177 {"id":7365845127484498177,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b46bdf47-fec6-4cb8-adf7-b8e8c3a06c13 {"id":"b46bdf47-fec6-4cb8-adf7-b8e8c3a06c13","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994477} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da2bca9ce b46bdf47-fec6-4cb8-adf7-b8e8c3a06c13 /tidb/cdc/default/default/upstream/7365845127484498177 {"id":7365845127484498177,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_content_compatible.cli.28840.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-canal-json-content-compatible?protocol=canal-json&enable-tidb-extension=true&content-compatible=true' Create changefeed successfully! ID: d81e9a42-4769-4814-995c-3f23a8820aa1 Info: {"upstream_id":7365845127484498177,"namespace":"default","id":"d81e9a42-4769-4814-995c-3f23a8820aa1","sink_uri":"kafka://127.0.0.1:9092/ticdc-canal-json-content-compatible?protocol=canal-json\u0026enable-tidb-extension=true\u0026content-compatible=true","create_time":"2024-05-06T19:21:20.427160837+08:00","start_ts":449575513049006082,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":true,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575513049006082,"checkpoint_ts":449575513049006082,"checkpoint_time":"2024-05-06 19:21:20.320"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 19232 30.9 0.5 4740276 2274192 ? Sl 19:20 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 19233 23.5 0.5 4691632 2222896 ? Sl 19:20 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 19234 23.2 0.5 4696756 2223264 ? Sl 19:20 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 19236 29.6 0.5 4726452 2264296 ? Sl 19:20 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2285 0 --:--:-- --:--:-- --:--:-- 2292 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:21:20.329","puller_resolved_ts":"2024-05-06 19:21:14.329","last_synced_ts":"2024-05-06 19:21:14.379","now_ts":"2024-05-06 19:21:21.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:21:20.329","puller_resolved_ts":"2024-05-06' '19:21:14.329","last_synced_ts":"2024-05-06' '19:21:14.379","now_ts":"2024-05-06' '19:21:21.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:21:20.329","puller_resolved_ts":"2024-05-06' '19:21:14.329","last_synced_ts":"2024-05-06' '19:21:14.379","now_ts":"2024-05-06' '19:21:21.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 [Mon May 6 19:21:26 CST 2024] <<<<<< START kafka consumer in canal_json_content_compatible case >>>>>> 7:21PM INF > Run case=sql/debezium/regression_test.sql table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 6-th check, retry later table test.finish_mark not exists for 7-th check, retry later table test.finish_mark not exists for 8-th check, retry later table test.finish_mark not exists for 9-th check, retry later table test.finish_mark not exists for 10-th check, retry later table test.finish_mark not exists for 11-th check, retry later table test.finish_mark exists check diff successfully table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-0kfbr io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3psjs-0kfbr] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-qql82 io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1872-wtdb3-qql82] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-lj2k4 io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1872-vd7dm-lj2k4] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) ERROR: Failed to launch pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-t6bnd io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[pingcap-tiflow-pull-cdc-integration-kafka-test-1872-8fv99-t6bnd] in namespace [jenkins-tiflow]. at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921) at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97) at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185) at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297) at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46) at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) at java.base/java.lang.Thread.run(Thread.java:829) table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later table test.finish_mark not exists for 5-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Mon May 6 19:22:28 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> 7:22PM INF > Run case=sql/debezium/skip_messages_test.sql 7:22PM INF > Run case=sql/debezium/strategy_test.sql =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... 7:22PM INF > Run case=sql/debezium/table_column_comment_test.sql start tidb cluster in /tmp/tidb_cdc_test/multi_topics Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... 7:22PM INF > Run case=sql/debezium/timestamp_column_test.sql Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 7:22PM INF > Run case=sql/debezium/tinyint_test.sql ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3691f14c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:30165, start at 2024-05-06 19:22:56.513027593 +0800 CST m=+5.407207696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:24:56.520 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:22:56.517 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:12:56.517 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 7:22PM INF > Run case=sql/debezium/topic_name_sanitization_test.sql VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3691f14c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:30165, start at 2024-05-06 19:22:56.513027593 +0800 CST m=+5.407207696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:24:56.520 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:22:56.517 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:12:56.517 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3691f23c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:30239, start at 2024-05-06 19:22:56.548276972 +0800 CST m=+5.392124622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:24:56.558 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:22:56.527 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:12:56.527 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.cli.31555.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575539629621249 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575539629621249 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:23:03 CST 2024] <<<<<< START cdc server in multi_topics case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.3158731589.out server --log-file /tmp/tidb_cdc_test/multi_topics/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2670 0 --:--:-- --:--:-- --:--:-- 2695 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:22:49.885","puller_resolved_ts":"2024-05-06 19:22:42.835","last_synced_ts":"2024-05-06 19:20:34.285","now_ts":"2024-05-06 19:22:51.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:22:49.885","puller_resolved_ts":"2024-05-06' '19:22:42.835","last_synced_ts":"2024-05-06' '19:20:34.285","now_ts":"2024-05-06' '19:22:51.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:22:49.885","puller_resolved_ts":"2024-05-06' '19:22:42.835","last_synced_ts":"2024-05-06' '19:20:34.285","now_ts":"2024-05-06' '19:22:51.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:23:06 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f750855-e0bf-441f-a96c-8cd4d8cc746e {"id":"5f750855-e0bf-441f-a96c-8cd4d8cc746e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da44f95d5 5f750855-e0bf-441f-a96c-8cd4d8cc746e /tidb/cdc/default/default/upstream/7365845583662118794 {"id":7365845583662118794,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f750855-e0bf-441f-a96c-8cd4d8cc746e {"id":"5f750855-e0bf-441f-a96c-8cd4d8cc746e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da44f95d5 5f750855-e0bf-441f-a96c-8cd4d8cc746e /tidb/cdc/default/default/upstream/7365845583662118794 {"id":7365845583662118794,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f750855-e0bf-441f-a96c-8cd4d8cc746e {"id":"5f750855-e0bf-441f-a96c-8cd4d8cc746e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da44f95d5 5f750855-e0bf-441f-a96c-8cd4d8cc746e /tidb/cdc/default/default/upstream/7365845583662118794 {"id":7365845583662118794,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.cli.31646.out cli changefeed create --start-ts=449575539629621249 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics/conf/changefeed.toml Create changefeed successfully! ID: e530e906-5dd9-4b91-beb8-e35d3446ac66 Info: {"upstream_id":7365845583662118794,"namespace":"default","id":"e530e906-5dd9-4b91-beb8-e35d3446ac66","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-05-06T19:23:06.73254627+08:00","start_ts":449575539629621249,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["workload.*"],"topic":"workload"},{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575539629621249,"checkpoint_ts":449575539629621249,"checkpoint_time":"2024-05-06 19:23:01.717"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x 7:23PM INF > Run case=sql/debezium/unsigned_integer_test.sql Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3693108c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:18821, start at 2024-05-06 19:23:14.866363401 +0800 CST m=+5.190849757 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:25:14.873 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:23:14.851 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:13:14.851 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3693108c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:18821, start at 2024-05-06 19:23:14.866363401 +0800 CST m=+5.190849757 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:25:14.873 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:23:14.851 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:13:14.851 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d3693113c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-397v7-76fbj, pid:18885, start at 2024-05-06 19:23:14.90169132 +0800 CST m=+5.155734172 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:25:14.909 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:23:14.895 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:13:14.895 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status_with_redo + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.20207.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575543990124545 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575543990124545 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575543990124545 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Mon May 6 19:23:19 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.2024120243.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:23:22 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cefc583c-982d-48e9-a51a-af4532c29fe2 {"id":"cefc583c-982d-48e9-a51a-af4532c29fe2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994600} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da499eed4 cefc583c-982d-48e9-a51a-af4532c29fe2 /tidb/cdc/default/default/upstream/7365845654277571066 {"id":7365845654277571066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cefc583c-982d-48e9-a51a-af4532c29fe2 {"id":"cefc583c-982d-48e9-a51a-af4532c29fe2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994600} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da499eed4 cefc583c-982d-48e9-a51a-af4532c29fe2 /tidb/cdc/default/default/upstream/7365845654277571066 {"id":7365845654277571066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cefc583c-982d-48e9-a51a-af4532c29fe2 {"id":"cefc583c-982d-48e9-a51a-af4532c29fe2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994600} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da499eed4 cefc583c-982d-48e9-a51a-af4532c29fe2 /tidb/cdc/default/default/upstream/7365845654277571066 {"id":7365845654277571066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575543990124545 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.20295.out cli changefeed create --start-ts=449575543990124545 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365845654277571066,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:23:23.359836683+08:00","start_ts":449575543990124545,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575543990124545,"checkpoint_ts":449575543990124545,"checkpoint_time":"2024-05-06 19:23:18.351"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8623 0 --:--:-- --:--:-- --:--:-- 8710 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:21:21.329","puller_resolved_ts":"2024-05-06 19:21:21.329","last_synced_ts":"2024-05-06 19:21:14.379","now_ts":"2024-05-06 19:23:31.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:21:21.329","puller_resolved_ts":"2024-05-06' '19:21:21.329","last_synced_ts":"2024-05-06' '19:21:14.379","now_ts":"2024-05-06' '19:23:31.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:21:21.329","puller_resolved_ts":"2024-05-06' '19:21:21.329","last_synced_ts":"2024-05-06' '19:21:14.379","now_ts":"2024-05-06' '19:23:31.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 10403 0 --:--:-- --:--:-- --:--:-- 10478 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:23:18.351","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:23:44.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:23:18.351","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:23:44.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:23:18.351","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:23:44.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36957cec0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:22813, start at 2024-05-06 19:23:54.584442787 +0800 CST m=+5.108996338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:25:54.590 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:23:54.555 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:13:54.555 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36957cec0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:22813, start at 2024-05-06 19:23:54.584442787 +0800 CST m=+5.108996338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:25:54.590 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:23:54.555 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:13:54.555 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36957f2c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:22891, start at 2024-05-06 19:23:54.711757772 +0800 CST m=+5.184853202 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:25:54.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:23:54.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:13:54.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + check_logs /tmp/tidb_cdc_test/synced_status_with_redo ++ date + echo '[Mon May 6 19:23:55 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>>' [Mon May 6 19:23:55 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>> + stop_tidb_cluster + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.24205.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575554411134977 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575554411134977 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575554411134977 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 19:23:59 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2424424246.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:24:02 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c018be2c-5978-4a28-8b88-f1cc295a8e31 {"id":"c018be2c-5978-4a28-8b88-f1cc295a8e31","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da5356bd5 c018be2c-5978-4a28-8b88-f1cc295a8e31 /tidb/cdc/default/default/upstream/7365845824868794435 {"id":7365845824868794435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c018be2c-5978-4a28-8b88-f1cc295a8e31 {"id":"c018be2c-5978-4a28-8b88-f1cc295a8e31","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da5356bd5 c018be2c-5978-4a28-8b88-f1cc295a8e31 /tidb/cdc/default/default/upstream/7365845824868794435 {"id":7365845824868794435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c018be2c-5978-4a28-8b88-f1cc295a8e31 {"id":"c018be2c-5978-4a28-8b88-f1cc295a8e31","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da5356bd5 c018be2c-5978-4a28-8b88-f1cc295a8e31 /tidb/cdc/default/default/upstream/7365845824868794435 {"id":7365845824868794435,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575554411134977 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.24303.out cli changefeed create --start-ts=449575554411134977 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365845824868794435,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:24:03.071494187+08:00","start_ts":449575554411134977,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575554411134977,"checkpoint_ts":449575554411134977,"checkpoint_time":"2024-05-06 19:23:58.104"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 22813 13.6 0.0 2361412 233072 ? Sl 19:23 0:03 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714994629469315323.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 22817 3.9 0.0 2679692 199836 ? Sl 19:23 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714994629472303781.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 22891 14.6 0.0 2855192 268648 ? Sl 19:23 0:03 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714994629519305668.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2537 0 --:--:-- --:--:-- --:--:-- 2557 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:24:10.504","puller_resolved_ts":"2024-05-06 19:24:04.504","last_synced_ts":"2024-05-06 19:24:04.604","now_ts":"2024-05-06 19:24:11.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:24:10.504","puller_resolved_ts":"2024-05-06' '19:24:04.504","last_synced_ts":"2024-05-06' '19:24:04.604","now_ts":"2024-05-06' '19:24:11.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:24:10.504","puller_resolved_ts":"2024-05-06' '19:24:04.504","last_synced_ts":"2024-05-06' '19:24:04.604","now_ts":"2024-05-06' '19:24:11.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 7:24PM INF > Run case=sql/dml.sql 7:25PM INF > All tests pass failed=0 passed=219 [Mon May 6 19:25:08 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 1 [Mon May 6 19:25:08 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 2 [Mon May 6 19:25:08 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 3 [Mon May 6 19:25:08 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> table test.table1 not exists for 1-th check, retry later table test.table1 not exists for 2-th check, retry later table test.table1 exists table test.table2 exists table test.table3 exists check diff successfully table test.table10 not exists for 1-th check, retry later table test.table10 exists table test.table20 exists check diff successfully [Mon May 6 19:25:19 CST 2024] <<<<<< run test case debezium success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:25:31 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:25:37 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Mon May 6 19:25:45 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 10 [Mon May 6 19:25:45 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 20 [Mon May 6 19:25:45 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: finish table test.finish not exists for 1-th check, retry later table test.finish not exists for 2-th check, retry later table test.finish exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:25:51 CST 2024] <<<<<< run test case multi_topics success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:26:06 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/avro_basic Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369e68980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:33215, start at 2024-05-06 19:26:20.757821639 +0800 CST m=+5.057734315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:28:20.763 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:26:20.760 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:16:20.760 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369e68980015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:33215, start at 2024-05-06 19:26:20.757821639 +0800 CST m=+5.057734315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:28:20.763 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:26:20.760 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:16:20.760 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369e6c640006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:33291, start at 2024-05-06 19:26:20.958343107 +0800 CST m=+5.204274998 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:28:20.965 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:26:20.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:16:20.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/avro_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/avro_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 49 100 24 100 25 314 327 --:--:-- --:--:-- --:--:-- 328 {"compatibility":"NONE"}+ pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.cli.34659.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575592764375041 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575592764375041 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:26:25 CST 2024] <<<<<< START cdc server in avro_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.3469734699.out server --log-file /tmp/tidb_cdc_test/avro_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/avro_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:26:28 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/137239e3-fca4-4614-829c-a66dc74c84e0 {"id":"137239e3-fca4-4614-829c-a66dc74c84e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994786} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da770bad2 137239e3-fca4-4614-829c-a66dc74c84e0 /tidb/cdc/default/default/upstream/7365846459666115239 {"id":7365846459666115239,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/137239e3-fca4-4614-829c-a66dc74c84e0 {"id":"137239e3-fca4-4614-829c-a66dc74c84e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994786} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da770bad2 137239e3-fca4-4614-829c-a66dc74c84e0 /tidb/cdc/default/default/upstream/7365846459666115239 {"id":7365846459666115239,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/137239e3-fca4-4614-829c-a66dc74c84e0 {"id":"137239e3-fca4-4614-829c-a66dc74c84e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994786} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da770bad2 137239e3-fca4-4614-829c-a66dc74c84e0 /tidb/cdc/default/default/upstream/7365846459666115239 {"id":7365846459666115239,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.cli.34746.out cli changefeed create --start-ts=449575592764375041 '--sink-uri=kafka://127.0.0.1:9092/ticdc-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true&avro-decimal-handling-mode=string&avro-bigint-unsigned-handling-mode=string' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/avro_basic/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 Create changefeed successfully! ID: e23d2468-3fad-4806-9498-7e30038ef808 Info: {"upstream_id":7365846459666115239,"namespace":"default","id":"e23d2468-3fad-4806-9498-7e30038ef808","sink_uri":"kafka://127.0.0.1:9092/ticdc-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true\u0026avro-decimal-handling-mode=string\u0026avro-bigint-unsigned-handling-mode=string","create_time":"2024-05-06T19:26:29.40901241+08:00","start_ts":449575592764375041,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575592764375041,"checkpoint_ts":449575592764375041,"checkpoint_time":"2024-05-06 19:26:24.410"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Mon May 6 19:26:30 CST 2024] <<<<<< START kafka consumer in avro_basic case >>>>>> schema registry uri found: http://127.0.0.1:8088 table test.finish_mark not exists for 1-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2646 0 --:--:-- --:--:-- --:--:-- 2662 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-06 19:26:20.655","puller_resolved_ts":"2024-05-06 19:26:13.655","last_synced_ts":"2024-05-06 19:24:04.604","now_ts":"2024-05-06 19:26:21.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:26:20.655","puller_resolved_ts":"2024-05-06' '19:26:13.655","last_synced_ts":"2024-05-06' '19:24:04.604","now_ts":"2024-05-06' '19:26:21.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-06' '19:26:20.655","puller_resolved_ts":"2024-05-06' '19:26:13.655","last_synced_ts":"2024-05-06' '19:24:04.604","now_ts":"2024-05-06' '19:26:21.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:26:38 CST 2024] <<<<<< run test case avro_basic success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369fd6a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:25545, start at 2024-05-06 19:26:44.156713121 +0800 CST m=+5.123808331 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:28:44.163 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:26:44.138 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:16:44.138 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369fd6a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:25545, start at 2024-05-06 19:26:44.156713121 +0800 CST m=+5.123808331 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:28:44.163 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:26:44.138 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:16:44.138 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d369fd7540005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-k2r9w-x4qr2, pid:25631, start at 2024-05-06 19:26:44.185326964 +0800 CST m=+5.103893694 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:28:44.191 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:26:44.181 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:16:44.181 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26999.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575598853455873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575598853455873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449575598853455873 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Mon May 6 19:26:49 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2703727039.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:26:52 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/505e2cf3-7af4-4294-b27c-8e5f9c8ceb8c {"id":"505e2cf3-7af4-4294-b27c-8e5f9c8ceb8c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994809} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da7cdccce 505e2cf3-7af4-4294-b27c-8e5f9c8ceb8c /tidb/cdc/default/default/upstream/7365846560212830253 {"id":7365846560212830253,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/505e2cf3-7af4-4294-b27c-8e5f9c8ceb8c {"id":"505e2cf3-7af4-4294-b27c-8e5f9c8ceb8c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994809} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da7cdccce 505e2cf3-7af4-4294-b27c-8e5f9c8ceb8c /tidb/cdc/default/default/upstream/7365846560212830253 {"id":7365846560212830253,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/505e2cf3-7af4-4294-b27c-8e5f9c8ceb8c {"id":"505e2cf3-7af4-4294-b27c-8e5f9c8ceb8c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994809} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da7cdccce 505e2cf3-7af4-4294-b27c-8e5f9c8ceb8c /tidb/cdc/default/default/upstream/7365846560212830253 {"id":7365846560212830253,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449575598853455873 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.27100.out cli changefeed create --start-ts=449575598853455873 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365846560212830253,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-06T19:26:52.63159016+08:00","start_ts":449575598853455873,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575598853455873,"checkpoint_ts":449575598853455873,"checkpoint_time":"2024-05-06 19:26:47.638"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/canal_json_handle_key_only Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... + set +x + sleep 20 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a104b0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:35861, start at 2024-05-06 19:27:03.478442727 +0800 CST m=+5.066630743 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:29:03.486 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:27:03.468 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:17:03.468 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a104b0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:35861, start at 2024-05-06 19:27:03.478442727 +0800 CST m=+5.066630743 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:29:03.486 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:27:03.468 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:17:03.468 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a107a80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:35930, start at 2024-05-06 19:27:03.684363783 +0800 CST m=+5.218355437 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:29:03.690 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:27:03.658 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:17:03.658 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.cli.37312.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575603933806593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575603933806593 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:27:08 CST 2024] <<<<<< START cdc server in canal_json_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.3733837340.out server --log-file /tmp/tidb_cdc_test/canal_json_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_handle_key_only/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:27:11 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a3a0abe-0448-4726-a67e-6f943afdcd49 {"id":"1a3a0abe-0448-4726-a67e-6f943afdcd49","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994828} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da8178ed0 1a3a0abe-0448-4726-a67e-6f943afdcd49 /tidb/cdc/default/default/upstream/7365846646538197764 {"id":7365846646538197764,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a3a0abe-0448-4726-a67e-6f943afdcd49 {"id":"1a3a0abe-0448-4726-a67e-6f943afdcd49","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994828} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da8178ed0 1a3a0abe-0448-4726-a67e-6f943afdcd49 /tidb/cdc/default/default/upstream/7365846646538197764 {"id":7365846646538197764,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a3a0abe-0448-4726-a67e-6f943afdcd49 {"id":"1a3a0abe-0448-4726-a67e-6f943afdcd49","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994828} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da8178ed0 1a3a0abe-0448-4726-a67e-6f943afdcd49 /tidb/cdc/default/default/upstream/7365846646538197764 {"id":7365846646538197764,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.cli.37397.out cli changefeed create --start-ts=449575603933806593 '--sink-uri=kafka://127.0.0.1:9092/canal-json-handle-key-only?protocol=canal-json&enable-tidb-extension=true&max-message-bytes=1000&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: 0015ed17-61fb-4a22-9cbe-ae5f70dfc12d Info: {"upstream_id":7365846646538197764,"namespace":"default","id":"0015ed17-61fb-4a22-9cbe-ae5f70dfc12d","sink_uri":"kafka://127.0.0.1:9092/canal-json-handle-key-only?protocol=canal-json\u0026enable-tidb-extension=true\u0026max-message-bytes=1000\u0026kafka-version=2.4.1","create_time":"2024-05-06T19:27:12.012753808+08:00","start_ts":449575603933806593,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"snappy","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575603933806593,"checkpoint_ts":449575603933806593,"checkpoint_time":"2024-05-06 19:27:07.018"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 10472 0 --:--:-- --:--:-- --:--:-- 10632 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-06 19:26:47.638","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-06 19:27:14.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:26:47.638","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:27:14.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-06' '19:26:47.638","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-06' '19:27:14.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:27:21 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Mon May 6 19:27:25 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Mon May 6 19:27:25 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh: line 1: 37429 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/open_protocol_handle_key_only Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a3a4c40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:38606, start at 2024-05-06 19:27:46.499778941 +0800 CST m=+5.075933457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:29:46.506 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:27:46.481 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:17:46.481 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a3a4c40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:38606, start at 2024-05-06 19:27:46.499778941 +0800 CST m=+5.075933457 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:29:46.506 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:27:46.481 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:17:46.481 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a3a7b00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:38679, start at 2024-05-06 19:27:46.692722444 +0800 CST m=+5.207705053 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:29:46.699 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:27:46.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:17:46.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.cli.39994.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575615209668609 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575615209668609 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:27:51 CST 2024] <<<<<< START cdc server in open_protocol_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.4002740029.out server --log-file /tmp/tidb_cdc_test/open_protocol_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/open_protocol_handle_key_only/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:27:54 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc9b9bae-b289-4c65-843c-d02e38823956 {"id":"fc9b9bae-b289-4c65-843c-d02e38823956","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994871} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da8bf93d0 fc9b9bae-b289-4c65-843c-d02e38823956 /tidb/cdc/default/default/upstream/7365846829635485511 {"id":7365846829635485511,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc9b9bae-b289-4c65-843c-d02e38823956 {"id":"fc9b9bae-b289-4c65-843c-d02e38823956","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994871} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da8bf93d0 fc9b9bae-b289-4c65-843c-d02e38823956 /tidb/cdc/default/default/upstream/7365846829635485511 {"id":7365846829635485511,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc9b9bae-b289-4c65-843c-d02e38823956 {"id":"fc9b9bae-b289-4c65-843c-d02e38823956","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994871} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da8bf93d0 fc9b9bae-b289-4c65-843c-d02e38823956 /tidb/cdc/default/default/upstream/7365846829635485511 {"id":7365846829635485511,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.cli.40081.out cli changefeed create --start-ts=449575615209668609 '--sink-uri=kafka://127.0.0.1:9092/open-protocol-handle-key-only?protocol=open-protocol&max-message-bytes=800&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: d538f8b7-ca8e-4746-a291-83b1ee086a04 Info: {"upstream_id":7365846829635485511,"namespace":"default","id":"d538f8b7-ca8e-4746-a291-83b1ee086a04","sink_uri":"kafka://127.0.0.1:9092/open-protocol-handle-key-only?protocol=open-protocol\u0026max-message-bytes=800\u0026kafka-version=2.4.1","create_time":"2024-05-06T19:27:55.021263391+08:00","start_ts":449575615209668609,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575615209668609,"checkpoint_ts":449575615209668609,"checkpoint_time":"2024-05-06 19:27:50.032"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:28:02 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh: line 1: 40112 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/canal_json_claim_check Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a62564001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:41300, start at 2024-05-06 19:28:27.529276126 +0800 CST m=+5.158613184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:30:27.536 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:28:27.531 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:18:27.531 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a62564001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:41300, start at 2024-05-06 19:28:27.529276126 +0800 CST m=+5.158613184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:30:27.536 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:28:27.531 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:18:27.531 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a624680015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:41368, start at 2024-05-06 19:28:27.460515418 +0800 CST m=+5.036396563 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:30:27.466 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:28:27.468 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:18:27.468 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.cli.42677.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575625944203265 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575625944203265 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:28:32 CST 2024] <<<<<< START cdc server in canal_json_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.4271242714.out server --log-file /tmp/tidb_cdc_test/canal_json_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_claim_check/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:28:35 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d62f7d0-a6dd-4807-9888-adb4e6950021 {"id":"6d62f7d0-a6dd-4807-9888-adb4e6950021","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994912} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da9617dd0 6d62f7d0-a6dd-4807-9888-adb4e6950021 /tidb/cdc/default/default/upstream/7365847006111481368 {"id":7365847006111481368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d62f7d0-a6dd-4807-9888-adb4e6950021 {"id":"6d62f7d0-a6dd-4807-9888-adb4e6950021","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994912} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da9617dd0 6d62f7d0-a6dd-4807-9888-adb4e6950021 /tidb/cdc/default/default/upstream/7365847006111481368 {"id":7365847006111481368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6d62f7d0-a6dd-4807-9888-adb4e6950021 {"id":"6d62f7d0-a6dd-4807-9888-adb4e6950021","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994912} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4da9617dd0 6d62f7d0-a6dd-4807-9888-adb4e6950021 /tidb/cdc/default/default/upstream/7365847006111481368 {"id":7365847006111481368,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.cli.42769.out cli changefeed create --start-ts=449575625944203265 '--sink-uri=kafka://127.0.0.1:9092/canal-json-claim-check?protocol=canal-json&enable-tidb-extension=true&max-message-bytes=1000&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/conf/changefeed.toml Create changefeed successfully! ID: b6eebf3c-36e0-46a8-8ea5-c6b3e53e66d1 Info: {"upstream_id":7365847006111481368,"namespace":"default","id":"b6eebf3c-36e0-46a8-8ea5-c6b3e53e66d1","sink_uri":"kafka://127.0.0.1:9092/canal-json-claim-check?protocol=canal-json\u0026enable-tidb-extension=true\u0026max-message-bytes=1000\u0026kafka-version=2.4.1","create_time":"2024-05-06T19:28:35.979257309+08:00","start_ts":449575625944203265,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/canal-json-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575625944203265,"checkpoint_ts":449575625944203265,"checkpoint_time":"2024-05-06 19:28:30.981"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:28:43 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh: line 1: 42802 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/open_protocol_claim_check Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a8b4bc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:43979, start at 2024-05-06 19:29:09.445815307 +0800 CST m=+5.104076132 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:31:09.453 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:29:09.423 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:19:09.423 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a8b4bc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:43979, start at 2024-05-06 19:29:09.445815307 +0800 CST m=+5.104076132 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:31:09.453 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:29:09.423 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:19:09.423 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36a8b6240015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:44045, start at 2024-05-06 19:29:09.550359278 +0800 CST m=+5.147702006 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:31:09.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:29:09.513 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:19:09.513 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.45346.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575636952154113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575636952154113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.45400.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449575638315302914 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449575638315302914 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Mon May 6 19:29:19 CST 2024] <<<<<< START cdc server in open_protocol_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.4542745429.out server --log-file /tmp/tidb_cdc_test/open_protocol_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/open_protocol_claim_check/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:29:22 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3fa33aff-9320-4f7d-8527-f3245d8f30ba {"id":"3fa33aff-9320-4f7d-8527-f3245d8f30ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994959} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4daa016b2b 3fa33aff-9320-4f7d-8527-f3245d8f30ba /tidb/cdc/default/default/upstream/7365847178938522810 {"id":7365847178938522810,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3fa33aff-9320-4f7d-8527-f3245d8f30ba {"id":"3fa33aff-9320-4f7d-8527-f3245d8f30ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994959} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4daa016b2b 3fa33aff-9320-4f7d-8527-f3245d8f30ba /tidb/cdc/default/default/upstream/7365847178938522810 {"id":7365847178938522810,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3fa33aff-9320-4f7d-8527-f3245d8f30ba {"id":"3fa33aff-9320-4f7d-8527-f3245d8f30ba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714994959} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4daa016b2b 3fa33aff-9320-4f7d-8527-f3245d8f30ba /tidb/cdc/default/default/upstream/7365847178938522810 {"id":7365847178938522810,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.45482.out cli changefeed create --start-ts=449575636952154113 --target-ts=449575638315302914 '--sink-uri=kafka://127.0.0.1:9092/open-protocol-claim-check?protocol=open-protocol&max-message-bytes=800&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/conf/changefeed.toml Create changefeed successfully! ID: 45cea8dc-3f2a-4bf4-9654-f5b96b471460 Info: {"upstream_id":7365847178938522810,"namespace":"default","id":"45cea8dc-3f2a-4bf4-9654-f5b96b471460","sink_uri":"kafka://127.0.0.1:9092/open-protocol-claim-check?protocol=open-protocol\u0026max-message-bytes=800\u0026kafka-version=2.4.1","create_time":"2024-05-06T19:29:23.158001725+08:00","start_ts":449575636952154113,"target_ts":449575638315302914,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"lz4","claim_check_storage_uri":"file:///tmp/open-protocol-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575636952154113,"checkpoint_ts":449575636952154113,"checkpoint_time":"2024-05-06 19:29:12.973"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:29:30 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh: line 1: 45510 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:29:41 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: kafka... <<================= [Mon May 6 19:29:44 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CCE2DB6419351B < X-Xss-Protection: 1; mode=block < Date: Mon, 06 May 2024 11:29:50 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Starting Downstream PD... Release Version: v8.2.0-alpha-15-gf83febabe Edition: Community Git Commit Hash: f83febabecb98b95b098fd31a3664178f8a6b437 Git Branch: master UTC Build Time: 2024-05-06 08:48:58 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 88099c95a3c0c13a827c0795c3d45070264665e4 Git Commit Branch: master UTC Build Time: 2024-05-06 06:37:19 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-84-g3ca57c1ff2 Edition: Community Git Commit Hash: 3ca57c1ff297b3ecabf6f7d509d86b0081d8e2b1 Git Branch: master UTC Build Time: 2024-05-06 10:31:02 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36abec64001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:46795, start at 2024-05-06 19:30:02.185429522 +0800 CST m=+5.060996300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:32:02.191 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:30:02.187 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:20:02.187 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36abec64001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:46795, start at 2024-05-06 19:30:02.185429522 +0800 CST m=+5.060996300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:32:02.191 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:30:02.187 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:20:02.187 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d36abef680014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-1872-3z6n6-hjwmn, pid:46864, start at 2024-05-06 19:30:02.35835939 +0800 CST m=+5.182745192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240506-19:32:02.365 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240506-19:30:02.330 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240506-19:20:02.330 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-17-g8e50de84e Edition: Community Git Commit Hash: 8e50de84e6d6ecdcc108990217b70b6bb3f50271 Git Branch: HEAD UTC Build Time: 2024-05-06 04:04:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-06 04:09:34 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e50de84e6d6ecdcc108990217b70b6bb3f50271"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-17-g8e50de84e"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Mon May 6 19:30:05 CST 2024] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.4832148323.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Mon, 06 May 2024 11:30:08 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96f4cbad-6b25-43ad-a48c-eced24c45751 {"id":"96f4cbad-6b25-43ad-a48c-eced24c45751","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714995005} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4daad97dc5 96f4cbad-6b25-43ad-a48c-eced24c45751 /tidb/cdc/default/default/upstream/7365847414425733650 {"id":7365847414425733650,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96f4cbad-6b25-43ad-a48c-eced24c45751 {"id":"96f4cbad-6b25-43ad-a48c-eced24c45751","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714995005} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4daad97dc5 96f4cbad-6b25-43ad-a48c-eced24c45751 /tidb/cdc/default/default/upstream/7365847414425733650 {"id":7365847414425733650,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96f4cbad-6b25-43ad-a48c-eced24c45751 {"id":"96f4cbad-6b25-43ad-a48c-eced24c45751","address":"127.0.0.1:8300","version":"v8.2.0-alpha-25-g58799adc6","git-hash":"58799adc6338905a3d6177e5829f44667d1aa09e","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1714995005} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4daad97dc5 96f4cbad-6b25-43ad-a48c-eced24c45751 /tidb/cdc/default/default/upstream/7365847414425733650 {"id":7365847414425733650,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7365847414425733650,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-9648?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:30:08.803179275+08:00","start_ts":449575650718384129,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575650718384129,"checkpoint_ts":449575650718384129,"checkpoint_time":"2024-05-06 19:30:05.487"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7365847414425733650,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-150?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:30:09.002333126+08:00","start_ts":449575650718384129,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575650718384129,"checkpoint_ts":449575650718384129,"checkpoint_time":"2024-05-06 19:30:05.487"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7365847414425733650,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-2269?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-06T19:30:09.203632558+08:00","start_ts":449575650718384129,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-25-g58799adc6","resolved_ts":449575650718384129,"checkpoint_ts":449575650718384129,"checkpoint_time":"2024-05-06 19:30:05.487"} [Mon May 6 19:30:09 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> [Mon May 6 19:30:09 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> [Mon May 6 19:30:09 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7365847414425733650, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449575652461641764, "checkpoint_time": "2024-05-06 19:30:12.137", "error": null }' + echo '{ "upstream_id": 7365847414425733650, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449575652461641764, "checkpoint_time": "2024-05-06 19:30:12.137", "error": null }' { "upstream_id": 7365847414425733650, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449575652461641764, "checkpoint_time": "2024-05-06 19:30:12.137", "error": null } ++ echo '{' '"upstream_id":' 7365847414425733650, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449575652461641764, '"checkpoint_time":' '"2024-05-06' '19:30:12.137",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365847414425733650, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449575652461641764, '"checkpoint_time":' '"2024-05-06' '19:30:12.137",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7365847414425733650, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449575655620739074, "checkpoint_time": "2024-05-06 19:30:24.188", "error": null }' + echo '{ "upstream_id": 7365847414425733650, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449575655620739074, "checkpoint_time": "2024-05-06 19:30:24.188", "error": null }' { "upstream_id": 7365847414425733650, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449575655620739074, "checkpoint_time": "2024-05-06 19:30:24.188", "error": null } ++ echo '{' '"upstream_id":' 7365847414425733650, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449575655620739074, '"checkpoint_time":' '"2024-05-06' '19:30:24.188",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365847414425733650, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449575655620739074, '"checkpoint_time":' '"2024-05-06' '19:30:24.188",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7365847414425733650, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449575651990044710, "checkpoint_time": "2024-05-06 19:30:10.338", "error": { "time": "2024-05-06T19:30:12.252999309+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7365847414425733650, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449575651990044710, "checkpoint_time": "2024-05-06 19:30:10.338", "error": { "time": "2024-05-06T19:30:12.252999309+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7365847414425733650, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449575651990044710, "checkpoint_time": "2024-05-06 19:30:10.338", "error": { "time": "2024-05-06T19:30:12.252999309+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365847414425733650, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575651990044710, '"checkpoint_time":' '"2024-05-06' '19:30:10.338",' '"error":' '{' '"time":' '"2024-05-06T19:30:12.252999309+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365847414425733650, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449575651990044710, '"checkpoint_time":' '"2024-05-06' '19:30:10.338",' '"error":' '{' '"time":' '"2024-05-06T19:30:12.252999309+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Mon May 6 19:30:26 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-1872/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS