Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_kafka_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-c456v-rvpvw --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "19b6ab89831de35c79bfc08b673820bb062d84fc" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-c456v" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git rev-list --no-walk e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 5 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=7fb05667-679b-4b04-9c52-db3cc40e4baa BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-c456v GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-236df335481f9578f70eb859f68d5ceead3aa27f6c9385fda1ec4c08661c0305 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql pingcap_tiflow_pull_cdc_integration_kafka_test_2156-c456v GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.0' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build880153104=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c456v-9llql bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 58632 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 033d66562f266733 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 20 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Fri May 31 09:37:19 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11081/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11081/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/11081/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/11081/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/entry/codec.go, cdc/entry/mounter.go, cdc/entry/mounter_test.go] [Pipeline] echo diff file not matched: cdc/entry/codec.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-350a2fb-e46df29) 190230016 bytes in 0.76 secs (250141577 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/.git/ .git HEAD is now at e46df29f4 auo adjust scheduler config POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master + 57b01d977...f59703ae3 refs/pull/11081/head -> origin/pr/11081/head (forced update) Previous HEAD position was e46df29f4 auo adjust scheduler config HEAD is now at 350a2fb45 *: tidy go mod (#11204) ๐Ÿšง Checkouting to base SHA:350a2fb4596b489af648982082e60ea5735178ff... HEAD is now at 350a2fb45 *: tidy go mod (#11204) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: 350a2fb4596b489af648982082e60ea5735178ff 350a2fb45 *: tidy go mod (#11204) 7fbae794a *: copy seahash from blainsmith.com/go/seahash (#11193) dce7d2372 model(ticdc): add one method to get the row changed event corresponding table id (#11195) ๐Ÿšง Pre-merge heads of pull requests to base SHA: 350a2fb4596b489af648982082e60ea5735178ff ... Merge made by the 'ort' strategy. cdc/entry/codec.go | 6 ++-- cdc/entry/mounter.go | 82 ++++++++++++++++++++++++++--------------------- cdc/entry/mounter_test.go | 2 +- 3 files changed, 50 insertions(+), 40 deletions(-) ๐Ÿงพ Pre-merged result: 18cf66054053b7e9a688825ac1a73321e0b6ee8d 18cf66054 Merge commit 'f59703ae3c583e094370068a597537791092805f' into HEAD f59703ae3 set as default 9db822722 fix ut โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache saved successfully (git/pingcap/tiflow/rev-350a2fb-f59703a) 190231040 bytes in 3.80 secs (50002712 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 903 0 --:--:-- --:--:-- --:--:-- 911 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 454 0 --:--:-- --:--:-- --:--:-- 455 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2372 0 --:--:-- --:--:-- --:--:-- 2411 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1269 0 --:--:-- --:--:-- --:--:-- 1281 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/35e09a2a36d227c6b45314fff0502ab93fab8aba/centos7/tidb-server.tar.gz 2024-05-31 17:37:51 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/35e09a2a36d227c6b45314fff0502ab93fab8aba/centos7/tidb-server.tar.gz [559748279/559748279] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/199b01792159e5d8e83ef419a5053401e998bb0e/centos7/pd-server.tar.gz 2024-05-31 17:38:06 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/199b01792159e5d8e83ef419a5053401e998bb0e/centos7/pd-server.tar.gz [192256388/192256388] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/7ccbe3144fd22b35eece56a8125734a7bc2f7502/centos7/tikv-server.tar.gz 2024-05-31 17:38:39 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/7ccbe3144fd22b35eece56a8125734a7bc2f7502/centos7/tikv-server.tar.gz [920321342/920321342] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/640f856eb25e246e81f6e3e4dec0c64c32a385f3/centos7/tiflash.tar.gz 2024-05-31 17:38:59 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/640f856eb25e246e81f6e3e4dec0c64c32a385f3/centos7/tiflash.tar.gz [457944843/457944843] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-31 17:39:04 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-31 17:39:07 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-31 17:39:08 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-31 17:39:09 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz 2024-05-31 17:39:14 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-79f1fd1e_linux-amd64.tar.gz [55896256/55896256] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-31 17:39:31 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 31 17:39 . drwxr-sr-x. 18 jenkins jenkins 4.0K May 31 17:39 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 31 17:39 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 31 17:39 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 31 16:19 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 31 16:19 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 31 16:19 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 31 16:19 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 31 15:40 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 31 16:20 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M May 31 16:35 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 31 16:35 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 31 16:35 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 31 16:35 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 31 16:35 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 31 16:35 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 12M May 31 16:35 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 31 16:35 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M May 31 16:35 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 216M May 31 16:38 tidb-server -rwxr-xr-x. 1 jenkins jenkins 382M May 31 16:19 tiflash -rwxr-xr-x. 1 jenkins jenkins 419M May 30 16:26 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 31 16:35 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 31 17:39 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 31 17:39 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 31 17:39 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 31 17:39 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 31 16:19 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 2023 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 31 16:19 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 2023 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 31 16:19 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 31 16:19 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 31 15:40 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 31 16:20 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M May 31 16:35 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 31 16:35 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 31 16:35 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 31 16:35 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 31 16:35 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 31 16:35 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 12M May 31 16:35 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 31 16:35 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M May 31 16:35 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 6 16:40 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 216M May 31 16:38 tidb-server -rwxr-xr-x. 1 jenkins jenkins 382M May 31 16:19 tiflash -rwxr-xr-x. 1 jenkins jenkins 419M May 30 16:26 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 31 16:35 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.1.5-2-geae7b5d Git Commit Hash: eae7b5d149e30a02a04ec6116e08b0a8f07fe4aa Git Branch: disable-cgo-for-sync-diff UTC Build Time: 2024-05-06 08:40:40 Go Version: go1.22.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 31 17:39 . drwxr-sr-x. 18 jenkins jenkins 4.0K May 31 17:39 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-75-g18cf66054" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 09:39:34" -X "github.com/pingcap/tiflow/pkg/version.GitHash=18cf66054053b7e9a688825ac1a73321e0b6ee8d" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-75-g18cf66054"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240520031014-397a460dd067 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading golang.org/x/net v0.25.0 go: downloading google.golang.org/grpc v1.63.2 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240520031014-397a460dd067 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240516031922-38e0dca30c8c go: downloading github.com/google/btree v1.1.2 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading golang.org/x/sys v0.20.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.20.0 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240507075657-2bd4573edde2 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.23.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/golang/mock v1.6.0 go: downloading golang.org/x/term v0.20.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/docker/go-units v0.5.0 go: downloading golang.org/x/text v0.15.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/99designs/keyring v1.2.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-75-g18cf66054" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 09:41:36" -X "github.com/pingcap/tiflow/pkg/version.GitHash=18cf66054053b7e9a688825ac1a73321e0b6ee8d" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-75-g18cf66054"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-75-g18cf66054" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 09:41:44" -X "github.com/pingcap/tiflow/pkg/version.GitHash=18cf66054053b7e9a688825ac1a73321e0b6ee8d" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-75-g18cf66054"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-75-g18cf66054" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 09:41:51" -X "github.com/pingcap/tiflow/pkg/version.GitHash=18cf66054053b7e9a688825ac1a73321e0b6ee8d" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-75-g18cf66054"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-75-g18cf66054" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 09:41:51" -X "github.com/pingcap/tiflow/pkg/version.GitHash=18cf66054053b7e9a688825ac1a73321e0b6ee8d" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-75-g18cf66054"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-75-g18cf66054" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 09:41:51" -X "github.com/pingcap/tiflow/pkg/version.GitHash=18cf66054053b7e9a688825ac1a73321e0b6ee8d" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-75-g18cf66054"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-75-g18cf66054" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 09:41:51" -X "github.com/pingcap/tiflow/pkg/version.GitHash=18cf66054053b7e9a688825ac1a73321e0b6ee8d" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-75-g18cf66054"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-75-g18cf66054" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 09:41:51" -X "github.com/pingcap/tiflow/pkg/version.GitHash=18cf66054053b7e9a688825ac1a73321e0b6ee8d" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-75-g18cf66054"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-75-g18cf66054" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-31 09:41:51" -X "github.com/pingcap/tiflow/pkg/version.GitHash=18cf66054053b7e9a688825ac1a73321e0b6ee8d" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.0 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-75-g18cf66054"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.1G drwxr-sr-x. 2 jenkins jenkins 4.0K May 31 17:46 . drwxr-sr-x. 18 jenkins jenkins 4.0K May 31 17:39 .. -rwxr-xr-x. 1 jenkins jenkins 214M May 31 17:46 cdc -rwxr-xr-x. 1 jenkins jenkins 346M May 31 17:46 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 31 17:42 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 31 17:42 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 31 17:42 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 31 17:42 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-75-g18cf66054 Git Commit Hash: 18cf66054053b7e9a688825ac1a73321e0b6ee8d Git Branch: HEAD UTC Build Time: 2024-05-31 09:41:51 Go Version: go version go1.21.0 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-test/rev-350a2fb-f59703a) 1171125248 bytes in 17.14 secs (68314566 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 31 17:47 . drwxr-sr-x. 18 jenkins jenkins 4.0K May 31 17:39 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 31 17:47 bin -rwxr-xr-x. 1 jenkins jenkins 214M May 31 17:46 cdc -rwxr-xr-x. 1 jenkins jenkins 346M May 31 17:46 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 31 17:42 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 31 17:42 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 31 17:42 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 31 17:47 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 31 17:47 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 31 17:47 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 31 17:47 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 31 17:47 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 31 17:47 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 31 17:47 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 31 17:47 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 31 17:47 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 31 17:47 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 31 17:47 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 31 17:47 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 31 17:47 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 31 17:47 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 31 17:42 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 31 17:47 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 31 17:47 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 31 17:47 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 31 17:47 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 31 17:47 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 31 17:47 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 12M May 31 17:47 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 31 17:47 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 31 17:47 share -rwxr-xr-x. 1 jenkins jenkins 32M May 31 17:47 stores-dump -rwxr-xr-x. 1 jenkins jenkins 133M May 31 17:47 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 216M May 31 17:47 tidb-server -rwxr-xr-x. 1 jenkins jenkins 382M May 31 17:47 tiflash -rwxr-xr-x. 1 jenkins jenkins 419M May 31 17:47 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 31 17:47 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 70.00 secs (52172450 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-2vnv8-l5ks2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "cd7ced17e39e9e9896ff5e21eb5f11b3d3c3ad72" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-2vnv8" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@41fb8b75; decorates RemoteLauncher[hudson.remoting.Channel@4a964c1d:JNLP4-connect connection from 10.233.123.209/10.233.123.209:56050] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fdkhs-8mktd --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "37f92afb06917710e4489547701ff295302acfc9" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fdkhs" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-9nv9v-3z3s6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ac8a48f8dcc92da138a40795bcc4cf68ed004cd3" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-9nv9v" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" [Pipeline] checkout Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-c379v-4ql2b --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9ed58b21a6c59c0f30442dc84080026e4cd70301" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-c379v" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2a5779fe; decorates RemoteLauncher[hudson.remoting.Channel@7d3f9b6:JNLP4-connect connection from 10.233.127.229/10.233.127.229:55668] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-28q8s-bw51h --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "fe90139c50b0d25a76ef3d4a35a987aae6ff33e9" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-28q8s" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@67ef545a; decorates RemoteLauncher[hudson.remoting.Channel@7a1ad006:JNLP4-connect connection from 10.233.67.250/10.233.67.250:53206] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7e9c08ee; decorates RemoteLauncher[hudson.remoting.Channel@4bdcb19c:JNLP4-connect connection from 10.233.86.219/10.233.86.219:38926] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] withEnv [Pipeline] { [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd [Pipeline] { [Pipeline] container No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@297daff; decorates RemoteLauncher[hudson.remoting.Channel@3147d4e8:JNLP4-connect connection from 10.233.90.126/10.233.90.126:48358] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] node [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-m7t4m-1tswr --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ae29085aed26f304bf1d905eb05f39fb811b46a6" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-m7t4m" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] checkout [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 The recommended git tool is: git [Pipeline] cache > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@ea12623; decorates RemoteLauncher[hudson.remoting.Channel@3de30090:JNLP4-connect connection from 10.233.68.97/10.233.68.97:45858] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-bxzm1-c51tr --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ff82782ac3ff3dc7c874bb7c2bf032d1486089f4" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-bxzm1" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-wlvl3-f4c9t --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "296b9786e28fa7937b826558f9f95dbfe2cac51a" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-wlvl3" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-lxbz5-4jpj5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5fe565b58b6448a888b72fb75da3743f1b0fc5f0" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-lxbz5" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-lchjd-bg550 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ef0229c51e9ee3094dd28d36fad05438cfa528b4" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-lchjd" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-p2gjp-2l6p4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f8294d9fce9c7efecb1189b31dccb0ffa54c8613" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-p2gjp" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_kafka_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-js4zt-p42sx --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b01918ccf507e93f987741fa690505d7398f54ef" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-js4zt" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 6.90 secs (529325948 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] sh [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k [Pipeline] checkout Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1 [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fw18n-j7mdr Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-kafka-test-2156-z6b6n-l09dl [Pipeline] stage [Pipeline] { (Test) The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] } [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // timeout [Pipeline] } No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@432ed07e; decorates RemoteLauncher[hudson.remoting.Channel@6faf60af:JNLP4-connect connection from 10.233.84.114/10.233.84.114:35958] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // container [Pipeline] sh Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4ba67add; decorates RemoteLauncher[hudson.remoting.Channel@2c553e35:JNLP4-connect connection from 10.233.97.130/10.233.97.130:38906] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1cb29f68; decorates RemoteLauncher[hudson.remoting.Channel@3d5bf47c:JNLP4-connect connection from 10.233.100.110/10.233.100.110:57404] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@70b874fe; decorates RemoteLauncher[hudson.remoting.Channel@6d8f42ce:JNLP4-connect connection from 10.233.73.51/10.233.73.51:37446] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1c5e094; decorates RemoteLauncher[hudson.remoting.Channel@6a89f828:JNLP4-connect connection from 10.233.69.96/10.233.69.96:55974] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@11bd8bb0; decorates RemoteLauncher[hudson.remoting.Channel@762153d5:JNLP4-connect connection from 10.233.106.98/10.233.106.98:50556] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=c098619a-2776-49f4-8597-dcd481420846 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-2vnv8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6 pingcap_tiflow_pull_cdc_integration_kafka_test_2156-2vnv8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:48:40 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-qppb5-whl3w --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bd6e3f44cfceb8a61495b71da60377160a9f8035" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-qppb5" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D489D984AB8E79 < X-Xss-Protection: 1; mode=block < Date: Fri, 31 May 2024 09:48:45 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Fri May 31 17:48:45 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1 is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-tr2x6-s29t2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "a197d6276cb20e03d8c542316a78c77942b07df2" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-tr2x6" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:48:48 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-z6b6n-l09dl is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-z6b6n-zd37s --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "28f4649db4594d389f1060c0e458eab8e2d55d74" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-z6b6n" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-z6b6n-l09dl" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-z6b6n-l09dl" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-z6b6n-l09dl" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-z6b6n-l09dl in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fv4zk-kp1ll --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "845d477363ab254d6cd0407e55623c879ac7b6f1" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fv4zk" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fw18n-j7mdr is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fw18n-v98lg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "4c8ba6b294579a086e0f6848e9dc12080c56f65c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fw18n" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fw18n-j7mdr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fw18n-j7mdr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fw18n-j7mdr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 8.08 secs (451812960 bytes/sec) [Pipeline] { Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fw18n-j7mdr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test [Pipeline] cache Agent pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn is provisioned from template pingcap_tiflow_pull_cdc_integration_kafka_test_2156-9jk86-3r018 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0c9125a12085be3bc92b97ca60ba9d17d948a469" jenkins/label: "pingcap_tiflow_pull_cdc_integration_kafka_test_2156-9jk86" name: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "wurstmeister/zookeeper" imagePullPolicy: "IfNotPresent" name: "zookeeper" resources: limits: cpu: "2000m" memory: "4Gi" requests: cpu: "2000m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - args: - "cat" image: "hub.pingcap.net/jenkins/golang-tini:1.21" imagePullPolicy: "Always" name: "golang" resources: limits: cpu: "12" memory: "32Gi" requests: cpu: "12" memory: "32Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_CREATE_TOPICS" value: "big-message-test:1:1" - name: "KAFKA_BROKER_ID" value: "1" - name: "KAFKA_SSL_KEYSTORE_PASSWORD" value: "test1234" - name: "KAFKA_ZOOKEEPER_CONNECT" value: "localhost:2181" - name: "KAFKA_MESSAGE_MAX_BYTES" value: "11534336" - name: "KAFKA_REPLICA_FETCH_MAX_BYTES" value: "11534336" - name: "KAFKA_ADVERTISED_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "ZK" value: "zk" - name: "KAFKA_SSL_KEYSTORE_LOCATION" value: "/tmp/kafka.server.keystore.jks" - name: "KAFKA_SSL_KEY_PASSWORD" value: "test1234" - name: "KAFKA_SSL_TRUSTSTORE_PASSWORD" value: "test1234" - name: "KAFKA_LISTENERS" value: "SSL://127.0.0.1:9093,PLAINTEXT://127.0.0.1:9092" - name: "KAFKA_SSL_TRUSTSTORE_LOCATION" value: "/tmp/kafka.server.truststore.jks" - name: "RACK_COMMAND" value: "curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.keystore.jks\ \ -o /tmp/kafka.server.keystore.jks && curl -sfL https://github.com/pingcap/tiflow/raw/6e62afcfecc4e3965d8818784327d4bf2600d9fa/tests/_certificates/kafka.server.truststore.jks\ \ -o /tmp/kafka.server.truststore.jks" image: "wurstmeister/kafka:2.12-2.4.1" imagePullPolicy: "IfNotPresent" name: "kafka" resources: limits: cpu: "4000m" memory: "6Gi" requests: cpu: "4000m" memory: "6Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "KAFKA_SERVER" value: "127.0.0.1:9092" - name: "ZOOKEEPER_SERVER" value: "127.0.0.1:2181" - name: "DOWNSTREAM_DB_HOST" value: "127.0.0.1" - name: "USE_FLAT_MESSAGE" value: "true" - name: "DOWNSTREAM_DB_PORT" value: "3306" - name: "DB_NAME" value: "test" image: "rustinliu/ticdc-canal-json-adapter:latest" imagePullPolicy: "IfNotPresent" name: "canal-adapter" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/tmp" name: "volume-0" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: requests: cpu: "200m" memory: "4Gi" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "MYSQL_ROOT_PASSWORD" value: "" - name: "MYSQL_USER" value: "mysqluser" - name: "MYSQL_PASSWORD" value: "mysqlpw" - name: "MYSQL_ALLOW_EMPTY_PASSWORD" value: "yes" - name: "MYSQL_TCP_PORT" value: "3310" image: "quay.io/debezium/example-mysql:2.4" imagePullPolicy: "IfNotPresent" name: "mysql" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "BOOTSTRAP_SERVERS" value: "127.0.0.1:9092" - name: "GROUP_ID" value: "1" - name: "CONFIG_STORAGE_TOPIC" value: "my_connect_configs" - name: "OFFSET_STORAGE_TOPIC" value: "my_connect_offsets" - name: "STATUS_STORAGE_TOPIC" value: "my_connect_statuses" image: "quay.io/debezium/connect:2.4" name: "connect" resources: requests: cpu: "200m" memory: "4Gi" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: {} name: "volume-0" - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:48:51 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:48:54 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 7.20 secs (507071897 bytes/sec) [Pipeline] { [Pipeline] cache * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D489DCF6604971 < X-Xss-Protection: 1; mode=block < Date: Fri, 31 May 2024 09:49:00 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Fri May 31 17:49:00 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:49:03 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3836d6b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6, pid:2177, start at 2024-05-31 17:49:16.117575203 +0800 CST m=+1.692292227 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:51:16.123 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:49:16.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:39:16.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3836d6b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6, pid:2177, start at 2024-05-31 17:49:16.117575203 +0800 CST m=+1.692292227 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:51:16.123 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:49:16.127 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:39:16.127 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3836d8c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6, pid:2249, start at 2024-05-31 17:49:16.242014738 +0800 CST m=+1.757587865 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:51:16.249 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:49:16.210 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:39:16.210 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri May 31 17:49:18 CST 2024] <<<<<< START cdc server in kafka_big_messages_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages_v2.35833585.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:49:21 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd9e79ec-18cc-47df-837c-b526728d4a46 {"id":"bd9e79ec-18cc-47df-837c-b526728d4a46","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717148959} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0d9d3d7f bd9e79ec-18cc-47df-837c-b526728d4a46 /tidb/cdc/default/default/upstream/7375098588078736871 {"id":7375098588078736871,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd9e79ec-18cc-47df-837c-b526728d4a46 {"id":"bd9e79ec-18cc-47df-837c-b526728d4a46","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717148959} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0d9d3d7f bd9e79ec-18cc-47df-837c-b526728d4a46 /tidb/cdc/default/default/upstream/7375098588078736871 {"id":7375098588078736871,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bd9e79ec-18cc-47df-837c-b526728d4a46 {"id":"bd9e79ec-18cc-47df-837c-b526728d4a46","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717148959} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0d9d3d7f bd9e79ec-18cc-47df-837c-b526728d4a46 /tidb/cdc/default/default/upstream/7375098588078736871 {"id":7375098588078736871,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: e1d65d2c-78af-41e2-a312-bfbe7f01351c Info: {"upstream_id":7375098588078736871,"namespace":"default","id":"e1d65d2c-78af-41e2-a312-bfbe7f01351c","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-05-31T17:49:22.098685819+08:00","start_ts":450140296649637889,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140296649637889,"checkpoint_ts":450140296649637889,"checkpoint_time":"2024-05-31 17:49:18.777"} [Fri May 31 17:49:22 CST 2024] <<<<<< START kafka consumer in kafka_big_messages_v2 case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 table kafka_big_messages.test exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 29.86 secs (122311977 bytes/sec) [Pipeline] { [Pipeline] cache check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:49:35 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383961340018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6, pid:5964, start at 2024-05-31 17:49:57.758505518 +0800 CST m=+1.714815882 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:51:57.767 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:49:57.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:39:57.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383961340018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6, pid:5964, start at 2024-05-31 17:49:57.758505518 +0800 CST m=+1.714815882 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:51:57.767 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:49:57.759 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:39:57.759 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383960640013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6, pid:6043, start at 2024-05-31 17:49:57.68289078 +0800 CST m=+1.567401975 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:51:57.688 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:49:57.657 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:39:57.657 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri May 31 17:50:00 CST 2024] <<<<<< START cdc server in multi_tables_ddl_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl_v2.73837385.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:50:03 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37a15b9b-4a3d-43bf-9153-b9aedef8ae44 {"id":"37a15b9b-4a3d-43bf-9153-b9aedef8ae44","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149000} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0e39cf83 37a15b9b-4a3d-43bf-9153-b9aedef8ae44 /tidb/cdc/default/default/upstream/7375098757125710316 {"id":7375098757125710316,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37a15b9b-4a3d-43bf-9153-b9aedef8ae44 {"id":"37a15b9b-4a3d-43bf-9153-b9aedef8ae44","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149000} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0e39cf83 37a15b9b-4a3d-43bf-9153-b9aedef8ae44 /tidb/cdc/default/default/upstream/7375098757125710316 {"id":7375098757125710316,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37a15b9b-4a3d-43bf-9153-b9aedef8ae44 {"id":"37a15b9b-4a3d-43bf-9153-b9aedef8ae44","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149000} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0e39cf83 37a15b9b-4a3d-43bf-9153-b9aedef8ae44 /tidb/cdc/default/default/upstream/7375098757125710316 {"id":7375098757125710316,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7375098757125710316,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-8213?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:50:03.727240389+08:00","start_ts":450140307563216897,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140307563216897,"checkpoint_ts":450140307563216897,"checkpoint_time":"2024-05-31 17:50:00.409"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7375098757125710316,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-25299?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:50:03.939782696+08:00","start_ts":450140307563216897,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140307563216897,"checkpoint_ts":450140307563216897,"checkpoint_time":"2024-05-31 17:50:00.409"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7375098757125710316,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-16431?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:50:04.125999303+08:00","start_ts":450140307563216897,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140307563216897,"checkpoint_ts":450140307563216897,"checkpoint_time":"2024-05-31 17:50:00.409"} [Fri May 31 17:50:04 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Fri May 31 17:50:04 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> [Fri May 31 17:50:04 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>> table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later table multi_tables_ddl_test.t55 not exists for 5-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 41.65 secs (87675910 bytes/sec) [Pipeline] { [Pipeline] cache table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7375098757125710316, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450140308782186552, "checkpoint_time": "2024-05-31 17:50:05.059", "error": null }' + echo '{ "upstream_id": 7375098757125710316, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450140308782186552, "checkpoint_time": "2024-05-31 17:50:05.059", "error": null }' { "upstream_id": 7375098757125710316, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450140308782186552, "checkpoint_time": "2024-05-31 17:50:05.059", "error": null } ++ echo '{' '"upstream_id":' 7375098757125710316, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450140308782186552, '"checkpoint_time":' '"2024-05-31' '17:50:05.059",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375098757125710316, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450140308782186552, '"checkpoint_time":' '"2024-05-31' '17:50:05.059",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7375098757125710316, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450140312452202503, "checkpoint_time": "2024-05-31 17:50:19.059", "error": null }' + echo '{ "upstream_id": 7375098757125710316, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450140312452202503, "checkpoint_time": "2024-05-31 17:50:19.059", "error": null }' { "upstream_id": 7375098757125710316, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450140312452202503, "checkpoint_time": "2024-05-31 17:50:19.059", "error": null } ++ echo '{' '"upstream_id":' 7375098757125710316, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450140312452202503, '"checkpoint_time":' '"2024-05-31' '17:50:19.059",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375098757125710316, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450140312452202503, '"checkpoint_time":' '"2024-05-31' '17:50:19.059",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7375098757125710316, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450140307563216897, "checkpoint_time": "2024-05-31 17:50:00.409", "error": { "time": "2024-05-31T17:50:05.256342767+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7375098757125710316, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450140307563216897, "checkpoint_time": "2024-05-31 17:50:00.409", "error": { "time": "2024-05-31T17:50:05.256342767+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7375098757125710316, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450140307563216897, "checkpoint_time": "2024-05-31 17:50:00.409", "error": { "time": "2024-05-31T17:50:05.256342767+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375098757125710316, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450140307563216897, '"checkpoint_time":' '"2024-05-31' '17:50:00.409",' '"error":' '{' '"time":' '"2024-05-31T17:50:05.256342767+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375098757125710316, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450140307563216897, '"checkpoint_time":' '"2024-05-31' '17:50:00.409",' '"error":' '{' '"time":' '"2024-05-31T17:50:05.256342767+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:50:21 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 7.73 secs (472493176 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + grep brokers + nc localhost 2181 + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] } [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@23fa63dc; decorates RemoteLauncher[hudson.remoting.Channel@1aa275b8:JNLP4-connect connection from 10.233.108.110/10.233.108.110:37414] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3988fe31; decorates RemoteLauncher[hudson.remoting.Channel@4736e4c3:JNLP4-connect connection from 10.233.105.10/10.233.105.10:49014] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@48d4ea44; decorates RemoteLauncher[hudson.remoting.Channel@303a0e91:JNLP4-connect connection from 10.233.107.165/10.233.107.165:55740] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@341460b5; decorates RemoteLauncher[hudson.remoting.Channel@785001a6:JNLP4-connect connection from 10.233.93.7/10.233.93.7:59778] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6a98a61f; decorates RemoteLauncher[hudson.remoting.Channel@cc9eaae:JNLP4-connect connection from 10.233.88.77/10.233.88.77:60898] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@756ff7ac; decorates RemoteLauncher[hudson.remoting.Channel@23edec5c:JNLP4-connect connection from 10.233.71.140/10.233.71.140:51402] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // timeout Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] sh > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) [Pipeline] sh Avoid second fetch Avoid second fetch Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=2e2a68d7-3919-4f2d-abd7-aaa3f2b2fb4a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fdkhs GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fdkhs GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: kafka... <<================= Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" [Pipeline] sh The 1 times to try to start tidb cluster... Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=c8b7427c-b8c5-42af-ba02-93c051647435 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-9nv9v GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-9nv9v pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:50:36 CST 2024] <<<<<< run test case http_api success! >>>>>> Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 [Pipeline] // container [Pipeline] // container > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 start tidb cluster in /tmp/tidb_cdc_test/multi_topics_v2 Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... [Pipeline] sh > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G00 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=d4c37476-4a60-4bcd-b69d-9c30bc4f31e9 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-28q8s GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f pingcap_tiflow_pull_cdc_integration_kafka_test_2156-28q8s GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=a5ea5e36-34f6-472c-842f-7305d38d0634 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-c379v GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-c379v pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:50:36 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=e525d55a-a7c7-47d3-9a17-828cee2afcc9 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-m7t4m GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-m7t4m pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Avoid second fetch Checking out Revision e64357d6ddd9be700fd5c8d87859c826b78da53a (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 Commit message: "Define code ownership using "OWNERS" files in the central CI config repo (#2976)" start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:50:39 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... > git checkout -f e64357d6ddd9be700fd5c8d87859c826b78da53a # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:50:39 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: kafka... <<================= VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c183c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6, pid:8931, start at 2024-05-31 17:50:42.209303157 +0800 CST m=+1.720234680 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:42.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:42.191 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:42.191 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c183c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6, pid:8931, start at 2024-05-31 17:50:42.209303157 +0800 CST m=+1.720234680 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:42.217 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:42.191 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:42.191 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c17700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-2vnv8-zdph6, pid:9007, start at 2024-05-31 17:50:42.176446982 +0800 CST m=+1.633610789 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:42.184 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:42.140 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:42.140 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: kafka... <<================= kafka downstream isn't support syncpoint record [Fri May 31 17:50:42 CST 2024] <<<<<< run test case syncpoint success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.10399.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c41740013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:1337, start at 2024-05-31 17:50:44.857527481 +0800 CST m=+1.778643968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:44.864 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:44.829 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:44.829 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c41740013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:1337, start at 2024-05-31 17:50:44.857527481 +0800 CST m=+1.778643968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:44.864 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:44.829 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:44.829 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c3fd80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:1414, start at 2024-05-31 17:50:44.771947282 +0800 CST m=+1.639647033 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:44.778 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:44.776 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:44.776 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:50:45 CST 2024] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... + set +x + tso='450140319276335105 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140319276335105 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:50:46 CST 2024] <<<<<< START cdc server in multi_topics_v2 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.1042810430.out server --log-file /tmp/tidb_cdc_test/multi_topics_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics_v2/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:50:46 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c64a0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:1402, start at 2024-05-31 17:50:47.091722672 +0800 CST m=+1.768295415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:47.098 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:47.080 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:47.080 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c64a0000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:1402, start at 2024-05-31 17:50:47.091722672 +0800 CST m=+1.768295415 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:47.098 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:47.080 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:47.080 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c63d8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:1480, start at 2024-05-31 17:50:47.043521808 +0800 CST m=+1.661275822 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:47.049 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:47.030 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:47.030 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri May 31 17:50:47 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.28002802.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 8.50 secs (429889909 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c75800008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:1346, start at 2024-05-31 17:50:48.167729496 +0800 CST m=+1.665938695 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:48.175 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:48.160 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:48.160 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c75800008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:1346, start at 2024-05-31 17:50:48.167729496 +0800 CST m=+1.665938695 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:48.175 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:48.160 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:48.160 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383c76440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:1425, start at 2024-05-31 17:50:48.243055943 +0800 CST m=+1.687733599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:52:48.251 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:50:48.209 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:40:48.209 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:50:49 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fd39b43-9e2a-4cef-9de7-2f8cb2b5957c {"id":"9fd39b43-9e2a-4cef-9de7-2f8cb2b5957c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149046} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0eef6189 9fd39b43-9e2a-4cef-9de7-2f8cb2b5957c /tidb/cdc/default/default/upstream/7375098958890534538 {"id":7375098958890534538,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fd39b43-9e2a-4cef-9de7-2f8cb2b5957c {"id":"9fd39b43-9e2a-4cef-9de7-2f8cb2b5957c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149046} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0eef6189 9fd39b43-9e2a-4cef-9de7-2f8cb2b5957c /tidb/cdc/default/default/upstream/7375098958890534538 {"id":7375098958890534538,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fd39b43-9e2a-4cef-9de7-2f8cb2b5957c {"id":"9fd39b43-9e2a-4cef-9de7-2f8cb2b5957c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149046} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0eef6189 9fd39b43-9e2a-4cef-9de7-2f8cb2b5957c /tidb/cdc/default/default/upstream/7375098958890534538 {"id":7375098958890534538,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.10488.out cli changefeed create --start-ts=450140319276335105 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/conf/changefeed.toml find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:50:49 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2804.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: e433e083-28c0-4b73-a3a4-c2b155e03419 Info: {"upstream_id":7375098958890534538,"namespace":"default","id":"e433e083-28c0-4b73-a3a4-c2b155e03419","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-05-31T17:50:50.049126985+08:00","start_ts":450140319276335105,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140319276335105,"checkpoint_ts":450140319276335105,"checkpoint_time":"2024-05-31 17:50:45.091"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:50:50 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bb34fd06-968c-4531-a6b6-29e938b06657 {"id":"bb34fd06-968c-4531-a6b6-29e938b06657","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149047} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0ef97d78 bb34fd06-968c-4531-a6b6-29e938b06657 /tidb/cdc/default/default/upstream/7375098969154388593 {"id":7375098969154388593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bb34fd06-968c-4531-a6b6-29e938b06657 {"id":"bb34fd06-968c-4531-a6b6-29e938b06657","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149047} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0ef97d78 bb34fd06-968c-4531-a6b6-29e938b06657 /tidb/cdc/default/default/upstream/7375098969154388593 {"id":7375098969154388593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bb34fd06-968c-4531-a6b6-29e938b06657 {"id":"bb34fd06-968c-4531-a6b6-29e938b06657","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149047} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0ef97d78 bb34fd06-968c-4531-a6b6-29e938b06657 /tidb/cdc/default/default/upstream/7375098969154388593 {"id":7375098969154388593,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: c0ce237c-c4a8-49f4-b02d-25d1a6c285d1 Info: {"upstream_id":7375098969154388593,"namespace":"default","id":"c0ce237c-c4a8-49f4-b02d-25d1a6c285d1","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-31T17:50:50.719157421+08:00","start_ts":450140319889489921,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140319889489921,"checkpoint_ts":450140319889489921,"checkpoint_time":"2024-05-31 17:50:47.430"} [Fri May 31 17:50:50 CST 2024] <<<<<< START kafka consumer in charset_gbk case >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2795.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140320544849921 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140320544849921 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:50:51 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.28412843.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x + tso='450140320854179841 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140320854179841 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:50:52 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.28212823.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:50:53 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:50:54 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1bbb51f2-888c-448a-b1fd-0cc5812e798e {"id":"1bbb51f2-888c-448a-b1fd-0cc5812e798e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149051} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0efe617e 1bbb51f2-888c-448a-b1fd-0cc5812e798e /tidb/cdc/default/default/upstream/7375098979605611869 {"id":7375098979605611869,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1bbb51f2-888c-448a-b1fd-0cc5812e798e {"id":"1bbb51f2-888c-448a-b1fd-0cc5812e798e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149051} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0efe617e 1bbb51f2-888c-448a-b1fd-0cc5812e798e /tidb/cdc/default/default/upstream/7375098979605611869 {"id":7375098979605611869,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1bbb51f2-888c-448a-b1fd-0cc5812e798e {"id":"1bbb51f2-888c-448a-b1fd-0cc5812e798e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149051} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0efe617e 1bbb51f2-888c-448a-b1fd-0cc5812e798e /tidb/cdc/default/default/upstream/7375098979605611869 {"id":7375098979605611869,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2891.out cli changefeed create --start-ts=450140320544849921 '--sink-uri=kafka://127.0.0.1:9092/ticdc-row-format-test-1012?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: b660aef0-6502-4ca9-9ffa-f610c57511ab Info: {"upstream_id":7375098979605611869,"namespace":"default","id":"b660aef0-6502-4ca9-9ffa-f610c57511ab","sink_uri":"kafka://127.0.0.1:9092/ticdc-row-format-test-1012?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:50:54.903074541+08:00","start_ts":450140320544849921,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140320544849921,"checkpoint_ts":450140320544849921,"checkpoint_time":"2024-05-31 17:50:49.930"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:50:55 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4521077-faac-4b64-8af3-5f5ffcf60512 {"id":"b4521077-faac-4b64-8af3-5f5ffcf60512","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149052} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0eff1686 b4521077-faac-4b64-8af3-5f5ffcf60512 /tidb/cdc/default/default/upstream/7375098984265181721 {"id":7375098984265181721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4521077-faac-4b64-8af3-5f5ffcf60512 {"id":"b4521077-faac-4b64-8af3-5f5ffcf60512","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149052} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0eff1686 b4521077-faac-4b64-8af3-5f5ffcf60512 /tidb/cdc/default/default/upstream/7375098984265181721 {"id":7375098984265181721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4521077-faac-4b64-8af3-5f5ffcf60512 {"id":"b4521077-faac-4b64-8af3-5f5ffcf60512","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149052} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0eff1686 b4521077-faac-4b64-8af3-5f5ffcf60512 /tidb/cdc/default/default/upstream/7375098984265181721 {"id":7375098984265181721,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2875.out cli changefeed create --start-ts=450140320854179841 '--sink-uri=kafka://127.0.0.1:9092/ticdc-foreign-key-test-6851?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: e4c2bb20-8f2c-46cb-92f5-b5afc03dcd11 Info: {"upstream_id":7375098984265181721,"namespace":"default","id":"e4c2bb20-8f2c-46cb-92f5-b5afc03dcd11","sink_uri":"kafka://127.0.0.1:9092/ticdc-foreign-key-test-6851?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:50:56.116241755+08:00","start_ts":450140320854179841,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140320854179841,"checkpoint_ts":450140320854179841,"checkpoint_time":"2024-05-31 17:50:51.110"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x [Fri May 31 17:50:56 CST 2024] <<<<<< START kafka consumer in row_format case >>>>>> table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later + set +x [Fri May 31 17:50:57 CST 2024] <<<<<< START kafka consumer in foreign_key case >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 6.97 secs (523907952 bytes/sec) [Pipeline] { [Pipeline] cache table charset_gbk_test1.t0 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383d41940013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:1713, start at 2024-05-31 17:51:01.260401399 +0800 CST m=+2.011067230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:01.267 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:01.270 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:01.270 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383d41940013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:1713, start at 2024-05-31 17:51:01.260401399 +0800 CST m=+2.011067230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:01.267 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:01.270 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:01.270 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383d4150000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:1777, start at 2024-05-31 17:51:01.215492445 +0800 CST m=+1.907571181 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:01.223 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:01.204 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:01.204 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table test.finish_mark not exists for 3-th check, retry later The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 table row_format.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... table test.finish_mark exists check table exists success check diff successfully Starting TLS TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table foreign_key.finish_mark not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:51:08 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> table row_format.finish_mark not exists for 2-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 8.30 secs (440051886 bytes/sec) [Pipeline] { [Pipeline] cache Starting TLS TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 3-th check, retry later table row_format.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383dd6e00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:3503, start at 2024-05-31 17:51:10.802940313 +0800 CST m=+1.866627759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:10.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:10.776 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:10.776 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3550.out cli tso query --pd=https://127.0.0.1:2579 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383ddaf00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:1791, start at 2024-05-31 17:51:11.044241501 +0800 CST m=+1.834055396 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:11.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:11.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:11.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383ddaf00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:1791, start at 2024-05-31 17:51:11.044241501 +0800 CST m=+1.834055396 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:11.051 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:11.036 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:11.036 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383dd93c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:1871, start at 2024-05-31 17:51:10.936978383 +0800 CST m=+1.666511791 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:10.943 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:10.927 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:10.927 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table row_format.finish_mark not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 4-th check, retry later + set +x + tso='450140326166790145 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140326166790145 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:51:12 CST 2024] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.36013603.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 5-th check, retry later table foreign_key.finish_mark not exists for 5-th check, retry later [Fri May 31 17:51:14 CST 2024] <<<<<< START cdc server in kafka_big_messages case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages.33363338.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:51:16 GMT < Content-Length: 1233 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/583a0fc3-113b-41df-b8e6-cf6fb169facf {"id":"583a0fc3-113b-41df-b8e6-cf6fb169facf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149073} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fce0f61c042 583a0fc3-113b-41df-b8e6-cf6fb169facf /tidb/cdc/default/default/upstream/7375099081412993468 {"id":7375099081412993468,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/583a0fc3-113b-41df-b8e6-cf6fb169facf {"id":"583a0fc3-113b-41df-b8e6-cf6fb169facf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149073} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fce0f61c042 583a0fc3-113b-41df-b8e6-cf6fb169facf /tidb/cdc/default/default/upstream/7375099081412993468 {"id":7375099081412993468,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/583a0fc3-113b-41df-b8e6-cf6fb169facf {"id":"583a0fc3-113b-41df-b8e6-cf6fb169facf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149073} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8fce0f61c042 583a0fc3-113b-41df-b8e6-cf6fb169facf /tidb/cdc/default/default/upstream/7375099081412993468 {"id":7375099081412993468,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x table row_format.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:51:17 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9ffab53-ec69-471c-8aa4-af963d214c8a {"id":"f9ffab53-ec69-471c-8aa4-af963d214c8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149074} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0f5d8785 f9ffab53-ec69-471c-8aa4-af963d214c8a /tidb/cdc/default/default/upstream/7375099078276559547 {"id":7375099078276559547,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9ffab53-ec69-471c-8aa4-af963d214c8a {"id":"f9ffab53-ec69-471c-8aa4-af963d214c8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149074} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0f5d8785 f9ffab53-ec69-471c-8aa4-af963d214c8a /tidb/cdc/default/default/upstream/7375099078276559547 {"id":7375099078276559547,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f9ffab53-ec69-471c-8aa4-af963d214c8a {"id":"f9ffab53-ec69-471c-8aa4-af963d214c8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149074} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0f5d8785 f9ffab53-ec69-471c-8aa4-af963d214c8a /tidb/cdc/default/default/upstream/7375099078276559547 {"id":7375099078276559547,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 6ac795a0-c18c-4be8-8799-109424b32046 Info: {"upstream_id":7375099078276559547,"namespace":"default","id":"6ac795a0-c18c-4be8-8799-109424b32046","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-05-31T17:51:17.748921446+08:00","start_ts":450140326969212929,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140326969212929,"checkpoint_ts":450140326969212929,"checkpoint_time":"2024-05-31 17:51:14.437"} [Fri May 31 17:51:17 CST 2024] <<<<<< START kafka consumer in kafka_big_messages case >>>>>> Starting generate kafka big messages... go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/atomic v1.11.0 table foreign_key.finish_mark not exists for 7-th check, retry later table row_format.finish_mark not exists for 7-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3663.out cli changefeed create --start-ts=450140326166790145 '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-5957?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7375099081412993468,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-5957?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:51:18.823742624+08:00","start_ts":450140326166790145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140326166790145,"checkpoint_ts":450140326166790145,"checkpoint_time":"2024-05-31 17:51:11.376"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 7.04 secs (519055873 bytes/sec) [Pipeline] { [Pipeline] cache table foreign_key.finish_mark not exists for 8-th check, retry later table row_format.finish_mark not exists for 8-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x [Fri May 31 17:51:20 CST 2024] <<<<<< START kafka consumer in cli_tls_with_auth case >>>>>> table test.simple not exists for 1-th check, retry later table row_format.finish_mark not exists for 9-th check, retry later table test.simple not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 9-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table row_format.finish_mark not exists for 10-th check, retry later table foreign_key.finish_mark exists check diff successfully table kafka_big_messages.test exists check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... table test.simple exists table test.`simple-dash` exists + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140329469804550, "checkpoint_time": "2024-05-31 17:51:23.976", "error": null }' + echo '{ "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140329469804550, "checkpoint_time": "2024-05-31 17:51:23.976", "error": null }' { "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140329469804550, "checkpoint_time": "2024-05-31 17:51:23.976", "error": null } ++ echo '{' '"upstream_id":' 7375099081412993468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140329469804550, '"checkpoint_time":' '"2024-05-31' '17:51:23.976",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375099081412993468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140329469804550, '"checkpoint_time":' '"2024-05-31' '17:51:23.976",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:51:25 CST 2024] <<<<<< run test case foreign_key success! >>>>>> Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc003687630} {Type:update Path:[Config SyncPointRetention] From: To:0xc003687638} {Type:update Path:[Config Consistent] From: To:0xc00109e0e0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3878.out cli changefeed --changefeed-id custom-changefeed-name pause check diff failed 2-th time, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:51:27 CST 2024] <<<<<< run test case row_format success! >>>>>> check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:51:29 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383f0cc40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:4015, start at 2024-05-31 17:51:30.652342199 +0800 CST m=+1.789173806 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:30.661 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:30.659 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:30.659 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383f0cc40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:4015, start at 2024-05-31 17:51:30.652342199 +0800 CST m=+1.789173806 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:30.661 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:30.659 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:30.659 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383f0b200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:4100, start at 2024-05-31 17:51:30.523975453 +0800 CST m=+1.604565696 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:30.529 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:30.504 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:30.504 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450140329745317889, "checkpoint_time": "2024-05-31 17:51:25.027", "error": null }' + echo '{ "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450140329745317889, "checkpoint_time": "2024-05-31 17:51:25.027", "error": null }' { "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450140329745317889, "checkpoint_time": "2024-05-31 17:51:25.027", "error": null } ++ echo '{' '"upstream_id":' 7375099081412993468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450140329745317889, '"checkpoint_time":' '"2024-05-31' '17:51:25.027",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7375099081412993468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450140329745317889, '"checkpoint_time":' '"2024-05-31' '17:51:25.027",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3947.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0038ca0c8} {Type:update Path:[Config SyncPointRetention] From: To:0xc0038ca0d8} {Type:update Path:[Config Consistent] From: To:0xc000d907e0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7375099081412993468,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-5957?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:51:18.823742624+08:00","start_ts":450140326166790145,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":0,"checkpoint_ts":450140329745317889,"checkpoint_time":"2024-05-31 17:51:25.027"} PASS coverage: 2.9% of statements in github.com/pingcap/tiflow/... + set +x [Fri May 31 17:51:33 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.55275529.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3974.out cli changefeed --changefeed-id custom-changefeed-name resume PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 11.85 secs (308148168 bytes/sec) [Pipeline] { [Pipeline] cache + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:51:36 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8afbc5c4-448f-4d73-a054-26b4560cfd2b {"id":"8afbc5c4-448f-4d73-a054-26b4560cfd2b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149093} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fa48281 8afbc5c4-448f-4d73-a054-26b4560cfd2b /tidb/cdc/default/default/upstream/7375099166215155975 {"id":7375099166215155975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8afbc5c4-448f-4d73-a054-26b4560cfd2b {"id":"8afbc5c4-448f-4d73-a054-26b4560cfd2b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149093} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fa48281 8afbc5c4-448f-4d73-a054-26b4560cfd2b /tidb/cdc/default/default/upstream/7375099166215155975 {"id":7375099166215155975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8afbc5c4-448f-4d73-a054-26b4560cfd2b {"id":"8afbc5c4-448f-4d73-a054-26b4560cfd2b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149093} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fa48281 8afbc5c4-448f-4d73-a054-26b4560cfd2b /tidb/cdc/default/default/upstream/7375099166215155975 {"id":7375099166215155975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5582.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-manager Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7375099166215155975,"namespace":"default","id":"ddl-manager","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:51:36.716092292+08:00","start_ts":450140332755255300,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140332755255300,"checkpoint_ts":450140332755255300,"checkpoint_time":"2024-05-31 17:51:36.509"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x [Fri May 31 17:51:38 CST 2024] <<<<<< START kafka consumer in ddl_manager case >>>>>> + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140332890783746, "checkpoint_time": "2024-05-31 17:51:37.026", "error": null }' + echo '{ "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140332890783746, "checkpoint_time": "2024-05-31 17:51:37.026", "error": null }' { "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140332890783746, "checkpoint_time": "2024-05-31 17:51:37.026", "error": null } ++ echo '{' '"upstream_id":' 7375099081412993468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140332890783746, '"checkpoint_time":' '"2024-05-31' '17:51:37.026",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375099081412993468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140332890783746, '"checkpoint_time":' '"2024-05-31' '17:51:37.026",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4046.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 450140333152927746 SinkURI: kafka://127.0.0.1:9092/ticdc-cli-test-5957?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4102.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-5957?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore [WARN] --tz is deprecated in changefeed settings. Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7375099081412993468,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-5957?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:51:44.390844754+08:00","start_ts":450140334765113346,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140334765113346,"checkpoint_ts":450140334765113346,"checkpoint_time":"2024-05-31 17:51:44.176"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x start tidb cluster in /tmp/tidb_cdc_test/kafka_compression Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383ffb480004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:4055, start at 2024-05-31 17:51:45.877255943 +0800 CST m=+1.718622303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:45.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:45.874 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:45.874 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383ffb480004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:4055, start at 2024-05-31 17:51:45.877255943 +0800 CST m=+1.718622303 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:45.884 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:45.874 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:45.874 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f383ffb200015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:4130, start at 2024-05-31 17:51:45.902716566 +0800 CST m=+1.693581171 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:45.910 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:45.914 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:45.914 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5577.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140335774367748, "checkpoint_time": "2024-05-31 17:51:48.026", "error": null }' + echo '{ "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140335774367748, "checkpoint_time": "2024-05-31 17:51:48.026", "error": null }' { "upstream_id": 7375099081412993468, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140335774367748, "checkpoint_time": "2024-05-31 17:51:48.026", "error": null } ++ echo '{' '"upstream_id":' 7375099081412993468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140335774367748, '"checkpoint_time":' '"2024-05-31' '17:51:48.026",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375099081412993468, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140335774367748, '"checkpoint_time":' '"2024-05-31' '17:51:48.026",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4190.out cli changefeed create --start-ts=450140326166790145 '--sink-uri=kafka://127.0.0.1:9093/ticdc-cli-test-ssl-29072?protocol=open-protocol&ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem&cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem&key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem&kafka-version=2.4.1&max-message-bytes=10485760&insecure-skip-verify=true' --tz=Asia/Shanghai + set +x + tso='450140335957606401 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140335957606401 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:51:50 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.56155617.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [WARN] --tz is deprecated in changefeed settings. Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 65c1579e-ac3c-4179-aa9e-48735020c553 Info: {"upstream_id":7375099081412993468,"namespace":"default","id":"65c1579e-ac3c-4179-aa9e-48735020c553","sink_uri":"kafka://127.0.0.1:9093/ticdc-cli-test-ssl-29072?protocol=open-protocol\u0026ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem\u0026key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760\u0026insecure-skip-verify=true","create_time":"2024-05-31T17:51:51.9393606+08:00","start_ts":450140326166790145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140326166790145,"checkpoint_ts":450140326166790145,"checkpoint_time":"2024-05-31 17:51:11.376"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384066580010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:5724, start at 2024-05-31 17:51:52.738282225 +0800 CST m=+1.712090250 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:52.744 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:52.726 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:52.726 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384066580010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:5724, start at 2024-05-31 17:51:52.738282225 +0800 CST m=+1.712090250 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:52.744 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:52.726 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:52.726 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384067cc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:5802, start at 2024-05-31 17:51:52.859931288 +0800 CST m=+1.783459232 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:52.867 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:52.870 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:52.870 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_compression/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_compression/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:51:53 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/773aa484-e24f-4a6c-8d0f-a230184ddadc {"id":"773aa484-e24f-4a6c-8d0f-a230184ddadc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149110} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fe825a0 773aa484-e24f-4a6c-8d0f-a230184ddadc /tidb/cdc/default/default/upstream/7375099234904720963 {"id":7375099234904720963,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/773aa484-e24f-4a6c-8d0f-a230184ddadc {"id":"773aa484-e24f-4a6c-8d0f-a230184ddadc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149110} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fe825a0 773aa484-e24f-4a6c-8d0f-a230184ddadc /tidb/cdc/default/default/upstream/7375099234904720963 {"id":7375099234904720963,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/773aa484-e24f-4a6c-8d0f-a230184ddadc {"id":"773aa484-e24f-4a6c-8d0f-a230184ddadc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149110} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fe825a0 773aa484-e24f-4a6c-8d0f-a230184ddadc /tidb/cdc/default/default/upstream/7375099234904720963 {"id":7375099234904720963,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5668.out cli changefeed create --start-ts=450140335957606401 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-12004?protocol=open-protocol&partition-num=4&kafka-client-id=ddl_puller_lag&kafka-version=2.4.1&max-message-bytes=10485760' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4219.out cli unsafe delete-service-gc-safepoint Create changefeed successfully! ID: ce8707df-2701-4190-bcf7-8d450d65aa54 Info: {"upstream_id":7375099234904720963,"namespace":"default","id":"ce8707df-2701-4190-bcf7-8d450d65aa54","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-ddl-puller-lag-test-12004?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=ddl_puller_lag\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:51:53.737128492+08:00","start_ts":450140335957606401,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140335957606401,"checkpoint_ts":450140335957606401,"checkpoint_time":"2024-05-31 17:51:48.725"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x [Fri May 31 17:51:55 CST 2024] <<<<<< START kafka consumer in ddl_puller_lag case >>>>>> + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4253.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri May 31 17:51:55 CST 2024] <<<<<< START cdc server in kafka_compression case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.71247126.out server --log-file /tmp/tidb_cdc_test/kafka_compression/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_compression/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 reset and all metadata truncated in PD! PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... wait process 5532 exit for 1-th time... wait process 5532 exit for 2-th time... wait process 5532 exit for 3-th time... wait process 5532 exit for 4-th time... wait process 5532 exit for 5-th time... wait process 5532 exit for 6-th time... wait process 5532 exit for 7-th time... wait process 5532 exit for 8-th time... + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3840a4a00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:4250, start at 2024-05-31 17:51:56.715860691 +0800 CST m=+1.731594595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:56.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:56.712 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:56.712 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3840a4a00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:4250, start at 2024-05-31 17:51:56.715860691 +0800 CST m=+1.731594595 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:56.722 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:56.712 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:56.712 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3840a6c80005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:4340, start at 2024-05-31 17:51:56.853966593 +0800 CST m=+1.804719139 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:53:56.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:51:56.850 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:41:56.850 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5532) - No such process wait process 5532 exit for 9-th time... process 5532 already exit [Fri May 31 17:51:57 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.56935695.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:51:58 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1927eddf-4802-4604-b609-9d3e2b45a23b {"id":"1927eddf-4802-4604-b609-9d3e2b45a23b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149115} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fff027e 1927eddf-4802-4604-b609-9d3e2b45a23b /tidb/cdc/default/default/upstream/7375099259894625402 {"id":7375099259894625402,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1927eddf-4802-4604-b609-9d3e2b45a23b {"id":"1927eddf-4802-4604-b609-9d3e2b45a23b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149115} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fff027e 1927eddf-4802-4604-b609-9d3e2b45a23b /tidb/cdc/default/default/upstream/7375099259894625402 {"id":7375099259894625402,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1927eddf-4802-4604-b609-9d3e2b45a23b {"id":"1927eddf-4802-4604-b609-9d3e2b45a23b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149115} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fff027e 1927eddf-4802-4604-b609-9d3e2b45a23b /tidb/cdc/default/default/upstream/7375099259894625402 {"id":7375099259894625402,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7180.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5660.out cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4349.out cli unsafe resolve-lock --region=18 + set +x + tso='450140338566201348 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140338566201348 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7224.out cli changefeed create --start-ts=450140338566201348 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip' -c gzip Create changefeed successfully! ID: gzip Info: {"upstream_id":7375099259894625402,"namespace":"default","id":"gzip","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=gzip","create_time":"2024-05-31T17:52:00.503065304+08:00","start_ts":450140338566201348,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140338566201348,"checkpoint_ts":450140338566201348,"checkpoint_time":"2024-05-31 17:51:58.676"} PASS PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + set +x + tso='450140338798198785 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140338798198785 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:52:01 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.56915693.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4380.out cli unsafe resolve-lock --region=18 --ts=450140338172985348 + set +x [Fri May 31 17:52:01 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:52:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7375099166215155975 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:51:36.716092292 +0800 CST StartTs:450140332755255300 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b10750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140332807946242} {CheckpointTs:450140333607223297 MinTableBarrierTs:450140333607223297 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe84978c-4861-4b94-83df-d1d06a7906a0 {"id":"fe84978c-4861-4b94-83df-d1d06a7906a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149117} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fa48547 fe84978c-4861-4b94-83df-d1d06a7906a0 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375099166215155975,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:51:36.716092292+08:00","start-ts":450140332755255300,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140332807946242} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450140333607223297,"min-table-barrier-ts":450140333607223297,"admin-job-type":0} /tidb/cdc/default/default/task/position/fe84978c-4861-4b94-83df-d1d06a7906a0/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375099166215155975 {"id":7375099166215155975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7375099166215155975 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:51:36.716092292 +0800 CST StartTs:450140332755255300 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b10750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140332807946242} {CheckpointTs:450140333607223297 MinTableBarrierTs:450140333607223297 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe84978c-4861-4b94-83df-d1d06a7906a0 {"id":"fe84978c-4861-4b94-83df-d1d06a7906a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149117} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fa48547 fe84978c-4861-4b94-83df-d1d06a7906a0 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375099166215155975,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:51:36.716092292+08:00","start-ts":450140332755255300,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140332807946242} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450140333607223297,"min-table-barrier-ts":450140333607223297,"admin-job-type":0} /tidb/cdc/default/default/task/position/fe84978c-4861-4b94-83df-d1d06a7906a0/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375099166215155975 {"id":7375099166215155975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7375099166215155975 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:51:36.716092292 +0800 CST StartTs:450140332755255300 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b10750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140332807946242} {CheckpointTs:450140333607223297 MinTableBarrierTs:450140333607223297 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe84978c-4861-4b94-83df-d1d06a7906a0 {"id":"fe84978c-4861-4b94-83df-d1d06a7906a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149117} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fa48547 fe84978c-4861-4b94-83df-d1d06a7906a0 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375099166215155975,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:51:36.716092292+08:00","start-ts":450140332755255300,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140332807946242} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450140333607223297,"min-table-barrier-ts":450140333607223297,"admin-job-type":0} /tidb/cdc/default/default/task/position/fe84978c-4861-4b94-83df-d1d06a7906a0/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375099166215155975 {"id":7375099166215155975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:52:01 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.57535755.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:52:02 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7375099166215155975 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:51:36.716092292 +0800 CST StartTs:450140332755255300 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b10750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140332807946242} {CheckpointTs:450140333607223297 MinTableBarrierTs:450140333607223297 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe84978c-4861-4b94-83df-d1d06a7906a0 {"id":"fe84978c-4861-4b94-83df-d1d06a7906a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149117} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fa48547 fe84978c-4861-4b94-83df-d1d06a7906a0 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375099166215155975,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:51:36.716092292+08:00","start-ts":450140332755255300,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140332807946242} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450140333738295310,"min-table-barrier-ts":450140333738295310,"admin-job-type":0} /tidb/cdc/default/default/task/position/fe84978c-4861-4b94-83df-d1d06a7906a0/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375099166215155975 {"id":7375099166215155975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7375099166215155975 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:51:36.716092292 +0800 CST StartTs:450140332755255300 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b10750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140332807946242} {CheckpointTs:450140333607223297 MinTableBarrierTs:450140333607223297 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe84978c-4861-4b94-83df-d1d06a7906a0 {"id":"fe84978c-4861-4b94-83df-d1d06a7906a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149117} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fa48547 fe84978c-4861-4b94-83df-d1d06a7906a0 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375099166215155975,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:51:36.716092292+08:00","start-ts":450140332755255300,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140332807946242} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450140333738295310,"min-table-barrier-ts":450140333738295310,"admin-job-type":0} /tidb/cdc/default/default/task/position/fe84978c-4861-4b94-83df-d1d06a7906a0/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375099166215155975 {"id":7375099166215155975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7375099166215155975 Namespace:default ID:ddl-manager SinkURI:kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:51:36.716092292 +0800 CST StartTs:450140332755255300 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003b10750 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140332807946242} {CheckpointTs:450140333607223297 MinTableBarrierTs:450140333607223297 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe84978c-4861-4b94-83df-d1d06a7906a0 {"id":"fe84978c-4861-4b94-83df-d1d06a7906a0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149117} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce0fa48547 fe84978c-4861-4b94-83df-d1d06a7906a0 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7375099166215155975,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"kafka://127.0.0.1:9092/ticdc-ddl-mamager-test-26646?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:51:36.716092292+08:00","start-ts":450140332755255300,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140332807946242} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":450140333738295310,"min-table-barrier-ts":450140333738295310,"admin-job-type":0} /tidb/cdc/default/default/task/position/fe84978c-4861-4b94-83df-d1d06a7906a0/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375099166215155975 {"id":7375099166215155975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/31 17:52:00.462 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:00.497 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:00.622 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:00.632 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:01.598 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:01.608 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]: syntax error: operand expected (error token is "[2024/05/31 17:52:00.462 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:00.497 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:00.622 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:00.632 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:01.598 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"] [2024/05/31 17:52:01.608 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses gzip compression algorithm"]") table test.gzip_finish_mark not exists for 1-th check, retry later + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 1605 45 --:--:-- --:--:-- --:--:-- 1618 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:52:04 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/460c2445-b463-474f-b2c7-a95b11a35ec5 {"id":"460c2445-b463-474f-b2c7-a95b11a35ec5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149121} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce100a778a 460c2445-b463-474f-b2c7-a95b11a35ec5 /tidb/cdc/default/default/upstream/7375099267382457689 {"id":7375099267382457689,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/460c2445-b463-474f-b2c7-a95b11a35ec5 {"id":"460c2445-b463-474f-b2c7-a95b11a35ec5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149121} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce100a778a 460c2445-b463-474f-b2c7-a95b11a35ec5 /tidb/cdc/default/default/upstream/7375099267382457689 {"id":7375099267382457689,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/460c2445-b463-474f-b2c7-a95b11a35ec5 {"id":"460c2445-b463-474f-b2c7-a95b11a35ec5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149121} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce100a778a 460c2445-b463-474f-b2c7-a95b11a35ec5 /tidb/cdc/default/default/upstream/7375099267382457689 {"id":7375099267382457689,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5750.out cli changefeed create --start-ts=450140338798198785 '--sink-uri=kafka://127.0.0.1:9092/ticdc-drop-tables-test-25579?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 0d1c6c46-61f7-499f-b13e-f09e5d36fe63 Info: {"upstream_id":7375099267382457689,"namespace":"default","id":"0d1c6c46-61f7-499f-b13e-f09e5d36fe63","sink_uri":"kafka://127.0.0.1:9092/ticdc-drop-tables-test-25579?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:52:04.478465525+08:00","start_ts":450140338798198785,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140338798198785,"checkpoint_ts":450140338798198785,"checkpoint_time":"2024-05-31 17:51:59.561"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 2-th check, retry later table test.gzip_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7346.out cli changefeed pause -c gzip PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x [Fri May 31 17:52:05 CST 2024] <<<<<< START kafka consumer in drop_many_tables case >>>>>> table drop_tables.c not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 3-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7376.out cli changefeed remove -c gzip } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 181 100 181 0 0 1369 0 --:--:-- --:--:-- --:--:-- 1371 { "version": "v8.2.0-alpha-75-g18cf66054", "git_hash": "18cf66054053b7e9a688825ac1a73321e0b6ee8d", "id": "c51298d7-0ce9-4e02-b270-dc7295337f90", "pid": 3606, "is_owner": true Changefeed remove successfully. ID: gzip CheckpointTs: 450140339549241373 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-gzip-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=gzip PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... }wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:52:08 CST 2024] <<<<<< run test case cli_tls_with_auth success! >>>>>> table drop_tables.c not exists for 2-th check, retry later + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7408.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 5-th check, retry later table drop_tables.c not exists for 3-th check, retry later + set +x + tso='450140341331820548 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140341331820548 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7444.out cli changefeed create --start-ts=450140341331820548 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy' -c snappy Create changefeed successfully! ID: snappy Info: {"upstream_id":7375099259894625402,"namespace":"default","id":"snappy","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=snappy","create_time":"2024-05-31T17:52:11.051883301+08:00","start_ts":450140341331820548,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140341331820548,"checkpoint_ts":450140341331820548,"checkpoint_time":"2024-05-31 17:52:09.226"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 6-th check, retry later + set +x [Fri May 31 17:52:12 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> table drop_tables.c not exists for 4-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/31 17:52:11.014 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:11.046 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:11.197 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:12.158 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:12.177 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:12.186 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]: syntax error: operand expected (error token is "[2024/05/31 17:52:11.014 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:11.046 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:11.197 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:12.158 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:12.177 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"] [2024/05/31 17:52:12.186 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses snappy compression algorithm"]") table test.snappy_finish_mark not exists for 1-th check, retry later table drop_tables.c not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later table test.snappy_finish_mark not exists for 2-th check, retry later table drop_tables.c exists check diff successfully table ddl_manager.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:52:17 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> table test.snappy_finish_mark exists Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 42.08 secs (86792556 bytes/sec) [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7571.out cli changefeed pause -c snappy [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 9-th check, retry later [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7605.out cli changefeed remove -c snappy + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) table ddl_manager.finish_mark not exists for 10-th check, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { Changefeed remove successfully. ID: snappy CheckpointTs: 450140342315122717 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-snappy-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=snappy PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7632.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 11-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x + tso='450140344766169095 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140344766169095 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7657.out cli changefeed create --start-ts=450140344766169095 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4' -c lz4 Create changefeed successfully! ID: lz4 Info: {"upstream_id":7375099259894625402,"namespace":"default","id":"lz4","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=lz4","create_time":"2024-05-31T17:52:24.224751457+08:00","start_ts":450140344766169095,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140344766169095,"checkpoint_ts":450140344766169095,"checkpoint_time":"2024-05-31 17:52:22.327"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 12-th check, retry later + set +x [Fri May 31 17:52:25 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 13-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/31 17:52:24.168 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:24.219 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:24.350 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:24.359 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:25.322 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:25.330 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]: syntax error: operand expected (error token is "[2024/05/31 17:52:24.168 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:24.219 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:24.350 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:24.359 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:25.322 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"] [2024/05/31 17:52:25.330 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses lz4 compression algorithm"]") table test.lz4_finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 14-th check, retry later [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN table test.lz4_finish_mark not exists for 2-th check, retry later [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] } [Pipeline] timeout Timeout set to expire in 45 min [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3842afa00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:5522, start at 2024-05-31 17:52:30.230425706 +0800 CST m=+1.902759820 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:54:30.237 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:52:30.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:42:30.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3842afa00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:5522, start at 2024-05-31 17:52:30.230425706 +0800 CST m=+1.902759820 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:54:30.237 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:52:30.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:42:30.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3842b0400014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:5601, start at 2024-05-31 17:52:30.268774257 +0800 CST m=+1.887539584 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:54:30.275 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:52:30.274 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:42:30.274 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 15-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] cache table test.lz4_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7817.out cli changefeed pause -c lz4 PASS table ddl_manager.finish_mark not exists for 16-th check, retry later coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Fri May 31 17:52:32 CST 2024] <<<<<< START cdc server in kafka_simple_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.69286930.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7839.out cli changefeed remove -c lz4 Changefeed remove successfully. ID: lz4 CheckpointTs: 450140345762316318 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-lz4-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=lz4 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 17-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:52:35 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eea6a237-887a-4978-a86e-af2dfc4640e8 {"id":"eea6a237-887a-4978-a86e-af2dfc4640e8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149153} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1092b180 eea6a237-887a-4978-a86e-af2dfc4640e8 /tidb/cdc/default/default/upstream/7375099422893607637 {"id":7375099422893607637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eea6a237-887a-4978-a86e-af2dfc4640e8 {"id":"eea6a237-887a-4978-a86e-af2dfc4640e8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149153} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1092b180 eea6a237-887a-4978-a86e-af2dfc4640e8 /tidb/cdc/default/default/upstream/7375099422893607637 {"id":7375099422893607637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eea6a237-887a-4978-a86e-af2dfc4640e8 {"id":"eea6a237-887a-4978-a86e-af2dfc4640e8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149153} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1092b180 eea6a237-887a-4978-a86e-af2dfc4640e8 /tidb/cdc/default/default/upstream/7375099422893607637 {"id":7375099422893607637,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.6976.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-2360?protocol=simple' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic/conf/changefeed.toml -c simple-basic + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7866.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: simple-basic Info: {"upstream_id":7375099422893607637,"namespace":"default","id":"simple-basic","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-2360?protocol=simple","create_time":"2024-05-31T17:52:36.210111968+08:00","start_ts":450140348359639045,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":5,"send_bootstrap_in_msg_count":100,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140348359639045,"checkpoint_ts":450140348359639045,"checkpoint_time":"2024-05-31 17:52:36.035"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 18-th check, retry later + set +x + tso='450140348383494146 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140348383494146 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.7902.out cli changefeed create --start-ts=450140348383494146 '--sink-uri=kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd' -c zstd + set +x Create changefeed successfully! ID: zstd Info: {"upstream_id":7375099259894625402,"namespace":"default","id":"zstd","sink_uri":"kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1\u0026compression=zstd","create_time":"2024-05-31T17:52:37.953427852+08:00","start_ts":450140348383494146,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140348383494146,"checkpoint_ts":450140348383494146,"checkpoint_time":"2024-05-31 17:52:36.126"} PASS Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... coverage: 2.5% of statements in github.com/pingcap/tiflow/... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 19-th check, retry later + set +x [Fri May 31 17:52:39 CST 2024] <<<<<< START kafka consumer in kafka_compression case >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 6.10 secs (598406739 bytes/sec) [Pipeline] { VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3843444c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:6952, start at 2024-05-31 17:52:39.726283624 +0800 CST m=+1.657388137 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:54:39.733 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:52:39.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:42:39.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3843444c001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:6952, start at 2024-05-31 17:52:39.726283624 +0800 CST m=+1.657388137 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:54:39.733 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:52:39.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:42:39.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384347600008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:7027, start at 2024-05-31 17:52:39.904728561 +0800 CST m=+1.781222408 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:54:39.911 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:52:39.896 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:42:39.896 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Pipeline] cache Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_compression/run.sh: line 22: [[: [2024/05/31 17:52:37.919 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:37.948 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:38.102 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:38.110 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:39.075 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:39.083 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]: syntax error: operand expected (error token is "[2024/05/31 17:52:37.919 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:37.948 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:38.102 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:38.110 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:39.075 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"] [2024/05/31 17:52:39.083 +08:00] [INFO] [sarama.go:96] ["Kafka producer uses zstd compression algorithm"]") table test.zstd_finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later table ddl_manager.finish_mark not exists for 21-th check, retry later [Fri May 31 17:52:42 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.84008402.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri May 31 17:52:42 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic case >>>>>> table test.finish_mark_for_ddl not exists for 1-th check, retry later table test.zstd_finish_mark not exists for 2-th check, retry later table test.zstd_finish_mark exists check diff successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.8024.out cli changefeed pause -c zstd table ddl_manager.finish_mark not exists for 22-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:52:45 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89dadda5-7db2-419e-997e-aeb3b92ca3a9 {"id":"89dadda5-7db2-419e-997e-aeb3b92ca3a9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149162} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce10b8a678 89dadda5-7db2-419e-997e-aeb3b92ca3a9 /tidb/cdc/default/default/upstream/7375099461153672755 {"id":7375099461153672755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89dadda5-7db2-419e-997e-aeb3b92ca3a9 {"id":"89dadda5-7db2-419e-997e-aeb3b92ca3a9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149162} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce10b8a678 89dadda5-7db2-419e-997e-aeb3b92ca3a9 /tidb/cdc/default/default/upstream/7375099461153672755 {"id":7375099461153672755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/89dadda5-7db2-419e-997e-aeb3b92ca3a9 {"id":"89dadda5-7db2-419e-997e-aeb3b92ca3a9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149162} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce10b8a678 89dadda5-7db2-419e-997e-aeb3b92ca3a9 /tidb/cdc/default/default/upstream/7375099461153672755 {"id":7375099461153672755,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table test.finish_mark_for_ddl not exists for 2-th check, retry later [Fri May 31 17:52:45 CST 2024] <<<<<< START kafka consumer in processor_stop_delay case >>>>>> table processor_stop_delay.t not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_compression.cli.8059.out cli changefeed remove -c zstd table ddl_manager.finish_mark not exists for 23-th check, retry later Changefeed remove successfully. ID: zstd CheckpointTs: 450140349366534174 SinkURI: kafka://127.0.0.1:9092/ticdc-kafka-compression-zstd-test?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1&compression=zstd PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table test.finish_mark_for_ddl not exists for 3-th check, retry later table processor_stop_delay.t not exists for 2-th check, retry later + set +x table ddl_manager.finish_mark not exists for 24-th check, retry later wait process cdc.test exit for 1-th time... table test.finish_mark_for_ddl not exists for 4-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:52:49 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> table processor_stop_delay.t exists check diff successfully table ddl_manager.finish_mark not exists for 25-th check, retry later table test.finish_mark_for_ddl not exists for 5-th check, retry later check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 26-th check, retry later table test.finish_mark_for_ddl exists + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.7102.out cli changefeed pause -c simple-basic PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic.cli.7133.out cli changefeed resume -c simple-basic table ddl_manager.finish_mark not exists for 27-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff failed 3-th time, retry later + set +x table ddl_manager.finish_mark not exists for 28-th check, retry later [Fri May 31 17:52:53 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 1 [Fri May 31 17:52:53 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 2 [Fri May 31 17:52:53 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 3 table test.table1 not exists for 1-th check, retry later table test.table1 not exists for 2-th check, retry later table test.table1 exists table test.table2 exists table test.table3 exists check diff successfully table test.table10 not exists for 1-th check, retry later check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 29-th check, retry later table test.finish_mark not exists for 1-th check, retry later table test.table10 exists table test.table20 exists check diff successfully check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 30-th check, retry later table test.finish_mark not exists for 2-th check, retry later check diff failed 6-th time, retry later table ddl_manager.finish_mark not exists for 31-th check, retry later table test.finish_mark not exists for 3-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 21.19 secs (172313762 bytes/sec) [Pipeline] { [Pipeline] cache check diff successfully wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:53:02 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> table ddl_manager.finish_mark not exists for 32-th check, retry later wait process cdc.test exit for 2-th time... table test.finish_mark not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:53:05 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 33-th check, retry later table test.finish_mark not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 34-th check, retry later table test.finish_mark not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 35-th check, retry later table test.finish_mark not exists for 7-th check, retry later table ddl_manager.finish_mark not exists for 36-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kafka_sink_error_resume Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table test.finish_mark not exists for 8-th check, retry later table ddl_manager.finish_mark not exists for 37-th check, retry later table test.finish_mark not exists for 9-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 38-th check, retry later table test.finish_mark not exists for 10-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 39-th check, retry later table test.finish_mark not exists for 11-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3845b7e80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:9233, start at 2024-05-31 17:53:19.88674025 +0800 CST m=+1.620470514 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:55:19.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:53:19.866 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:43:19.866 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3845b7e80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:9233, start at 2024-05-31 17:53:19.88674025 +0800 CST m=+1.620470514 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:55:19.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:53:19.866 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:43:19.866 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3845ba2c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:9309, start at 2024-05-31 17:53:20.053198168 +0800 CST m=+1.734288300 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:55:20.060 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:53:20.061 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:43:20.061 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_sink_error_resume/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 40-th check, retry later table test.finish_mark not exists for 12-th check, retry later table ddl_manager.finish_mark not exists for 41-th check, retry later [Fri May 31 17:53:22 CST 2024] <<<<<< START cdc server in kafka_sink_error_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_sink_error_resume.1067710679.out server --log-file /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_sink_error_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table test.finish_mark not exists for 13-th check, retry later table ddl_manager.finish_mark not exists for 42-th check, retry later table test.finish_mark not exists for 14-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:53:25 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3136f63c-72d7-4c74-b25f-81d195aaf3f5 {"id":"3136f63c-72d7-4c74-b25f-81d195aaf3f5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149202} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1151d67f 3136f63c-72d7-4c74-b25f-81d195aaf3f5 /tidb/cdc/default/default/upstream/7375099638049066230 {"id":7375099638049066230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3136f63c-72d7-4c74-b25f-81d195aaf3f5 {"id":"3136f63c-72d7-4c74-b25f-81d195aaf3f5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149202} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1151d67f 3136f63c-72d7-4c74-b25f-81d195aaf3f5 /tidb/cdc/default/default/upstream/7375099638049066230 {"id":7375099638049066230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3136f63c-72d7-4c74-b25f-81d195aaf3f5 {"id":"3136f63c-72d7-4c74-b25f-81d195aaf3f5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149202} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1151d67f 3136f63c-72d7-4c74-b25f-81d195aaf3f5 /tidb/cdc/default/default/upstream/7375099638049066230 {"id":7375099638049066230,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:53:26 CST 2024] <<<<<< START kafka consumer in kafka_sink_error_resume case >>>>>> table ddl_manager.finish_mark not exists for 43-th check, retry later table test.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_status 127.0.0.1:8300 8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1140 0 --:--:-- --:--:-- --:--:-- 1146 + info='{"state":"normal","resolved_ts":450140361737371651,"checkpoint_ts":450140361737371651}' + echo '{"state":"normal","resolved_ts":450140361737371651,"checkpoint_ts":450140361737371651}' {"state":"normal","resolved_ts":450140361737371651,"checkpoint_ts":450140361737371651} ++ echo '{"state":"normal","resolved_ts":450140361737371651,"checkpoint_ts":450140361737371651}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 44-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38464fc40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:10024, start at 2024-05-31 17:53:29.608112909 +0800 CST m=+1.776395913 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:55:29.614 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:53:29.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:43:29.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38464fc40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:10024, start at 2024-05-31 17:53:29.608112909 +0800 CST m=+1.776395913 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:55:29.614 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:53:29.585 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:43:29.585 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38464fc80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-28q8s-tvt8f, pid:10099, start at 2024-05-31 17:53:29.615329164 +0800 CST m=+1.729649365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:55:29.622 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:53:29.586 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:43:29.586 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 1-th check, retry later check_changefeed_status 127.0.0.1:8300 8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a warning last_warning kafka sink injected error + endpoint=127.0.0.1:8300 + changefeed_id=8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a + expected_state=warning + field=last_warning + error_pattern=kafka ++ curl 127.0.0.1:8300/api/v2/changefeeds/8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 4078 0 --:--:-- --:--:-- --:--:-- 4135 + info='{"state":"warning","resolved_ts":450140362327195649,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450140362327195649,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450140362327195649,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450140362327195649,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":450140362327195649,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .last_warning.message + error_msg='kafka sink injected error' + [[ ! kafka sink injected error =~ kafka ]] run task successfully check_changefeed_status 127.0.0.1:8300 8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a normal + endpoint=127.0.0.1:8300 + changefeed_id=8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table ddl_manager.finish_mark not exists for 45-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 2951 0 --:--:-- --:--:-- --:--:-- 2975 + info='{"state":"warning","resolved_ts":450140362327195649,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450140362327195649,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450140362327195649,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450140362327195649,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11481.out cli tso query --pd=http://127.0.0.1:2379 [Fri May 31 17:53:31 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 10 [Fri May 31 17:53:31 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: 20 [Fri May 31 17:53:31 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>> schema registry uri found: finish table test.finish not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 46-th check, retry later check_changefeed_status 127.0.0.1:8300 8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a normal + endpoint=127.0.0.1:8300 + changefeed_id=8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 3941 0 --:--:-- --:--:-- --:--:-- 4000 + info='{"state":"warning","resolved_ts":450140363113365506,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450140363113365506,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450140363113365506,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450140363113365506,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later table test.finish not exists for 2-th check, retry later + set +x + tso='450140363144560641 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140363144560641 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:53:33 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1152111523.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 47-th check, retry later table test.finish exists check diff successfully table test.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:53:37 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> check_changefeed_status 127.0.0.1:8300 8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a normal + endpoint=127.0.0.1:8300 + changefeed_id=8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 4000 0 --:--:-- --:--:-- --:--:-- 4066 + info='{"state":"warning","resolved_ts":450140364161941508,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450140364161941508,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450140364161941508,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450140364161941508,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 3-th time, retry later table ddl_manager.finish_mark not exists for 48-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:53:36 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3be7b19-19fd-4a47-aebe-1d38b2ca9624 {"id":"d3be7b19-19fd-4a47-aebe-1d38b2ca9624","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149214} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce11776487 d3be7b19-19fd-4a47-aebe-1d38b2ca9624 /tidb/cdc/default/default/upstream/7375099677071616963 {"id":7375099677071616963,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3be7b19-19fd-4a47-aebe-1d38b2ca9624 {"id":"d3be7b19-19fd-4a47-aebe-1d38b2ca9624","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149214} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce11776487 d3be7b19-19fd-4a47-aebe-1d38b2ca9624 /tidb/cdc/default/default/upstream/7375099677071616963 {"id":7375099677071616963,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3be7b19-19fd-4a47-aebe-1d38b2ca9624 {"id":"d3be7b19-19fd-4a47-aebe-1d38b2ca9624","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149214} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce11776487 d3be7b19-19fd-4a47-aebe-1d38b2ca9624 /tidb/cdc/default/default/upstream/7375099677071616963 {"id":7375099677071616963,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11591.out cli changefeed create --start-ts=450140363144560641 '--sink-uri=kafka://127.0.0.1:9092/ticdc-partition-table-test-27971?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 02c7b065-6c7f-41c4-8f66-0095026530c7 Info: {"upstream_id":7375099677071616963,"namespace":"default","id":"02c7b065-6c7f-41c4-8f66-0095026530c7","sink_uri":"kafka://127.0.0.1:9092/ticdc-partition-table-test-27971?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:53:37.332949898+08:00","start_ts":450140363144560641,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140363144560641,"checkpoint_ts":450140363144560641,"checkpoint_time":"2024-05-31 17:53:32.435"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 5-th check, retry later + set +x [Fri May 31 17:53:38 CST 2024] <<<<<< START kafka consumer in partition_table case >>>>>> table ddl_manager.finish_mark not exists for 49-th check, retry later table test.finish_mark not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 50-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 30.07 secs (121467384 bytes/sec) [Pipeline] { [Pipeline] cache table test.finish_mark not exists for 7-th check, retry later table ddl_manager.finish_mark not exists for 51-th check, retry later check_changefeed_status 127.0.0.1:8300 8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a normal + endpoint=127.0.0.1:8300 + changefeed_id=8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 244 100 244 0 0 3914 0 --:--:-- --:--:-- --:--:-- 3935 + info='{"state":"warning","resolved_ts":450140365735067652,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' + echo '{"state":"warning","resolved_ts":450140365735067652,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}}' {"state":"warning","resolved_ts":450140365735067652,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka sink injected error"}} ++ echo '{"state":"warning","resolved_ts":450140365735067652,"checkpoint_ts":450140361868181534,"last_warning":{"time":"2024-05-31T17:53:29.925924581+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrProcessorUnknown","message":"kafka' sink injected 'error"}}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 4-th time, retry later table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later table test.finish_mark not exists for 8-th check, retry later table ddl_manager.finish_mark not exists for 52-th check, retry later table partition_table.t2 not exists for 2-th check, retry later table test.finish_mark not exists for 9-th check, retry later table ddl_manager.finish_mark not exists for 53-th check, retry later table partition_table.t2 not exists for 3-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 54-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=kafka + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run kafka + '[' kafka '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Fri May 31 17:53:48 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Fri May 31 17:53:48 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 6.12 secs (597033496 bytes/sec) [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] cache cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:53:49 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> table partition_table.t2 not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 55-th check, retry later check_changefeed_status 127.0.0.1:8300 8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a normal + endpoint=127.0.0.1:8300 + changefeed_id=8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a + expected_state=normal + field= + error_pattern= ++ curl 127.0.0.1:8300/api/v2/changefeeds/8ad0263e-ee70-4162-8fbb-8fe9ef0ce72a/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1429 0 --:--:-- --:--:-- --:--:-- 1433 + info='{"state":"normal","resolved_ts":450140367832219652,"checkpoint_ts":450140367832219652}' + echo '{"state":"normal","resolved_ts":450140367832219652,"checkpoint_ts":450140367832219652}' {"state":"normal","resolved_ts":450140367832219652,"checkpoint_ts":450140367832219652} ++ echo '{"state":"normal","resolved_ts":450140367832219652,"checkpoint_ts":450140367832219652}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z '' ]] ++ echo '{"state":"normal","resolved_ts":450140367832219652,"checkpoint_ts":450140367832219652}' ++ jq -r .last_error + error_msg=null + [[ ! null == \n\u\l\l ]] ++ echo '{"state":"normal","resolved_ts":450140367832219652,"checkpoint_ts":450140367832219652}' ++ jq -r .last_warning + error_msg=null + [[ ! null == \n\u\l\l ]] + exit 0 run task successfully table kafka_sink_error_resume.t1 exists table kafka_sink_error_resume.t2 exists check diff successfully check diff failed 1-th time, retry later table partition_table.t2 not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:53:51 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> table ddl_manager.finish_mark not exists for 56-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:53:55 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> table partition_table.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 58-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri May 31 17:53:57 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> table partition_table.finish_mark not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:53:58 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> table ddl_manager.finish_mark not exists for 59-th check, retry later table partition_table.finish_mark not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 60-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m table partition_table.finish_mark not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 61-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 11.10 secs (328901936 bytes/sec) [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] sh table partition_table.finish_mark not exists for 6-th check, retry later [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] sh table ddl_manager.finish_mark not exists for 62-th check, retry later [Pipeline] sh start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_basic_avro Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] // timeout [Pipeline] } [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + awk {$1=$1;print} + + grepgrep -F brokers -w /brokers/ids/1 /brokers/ids/1 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow [Pipeline] { + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 [Pipeline] cache table partition_table.finish_mark not exists for 7-th check, retry later table ddl_manager.finish_mark not exists for 63-th check, retry later Verifying downstream PD is started... table partition_table.finish_mark not exists for 8-th check, retry later + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 64-th check, retry later table partition_table.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 65-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:54:11 CST 2024] <<<<<< run test case partition_table success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3848ef800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:8450, start at 2024-05-31 17:54:12.601473681 +0800 CST m=+1.885336414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:12.608 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:12.576 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:12.576 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3848ef800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:8450, start at 2024-05-31 17:54:12.601473681 +0800 CST m=+1.885336414 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:12.608 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:12.576 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:12.576 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3848f10c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:8521, start at 2024-05-31 17:54:12.690378261 +0800 CST m=+1.912939930 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:12.697 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:12.675 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:12.675 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 66-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_basic_avro/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 67-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:54:13 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> [Fri May 31 17:54:15 CST 2024] <<<<<< START cdc server in kafka_simple_basic_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.98969898.out server --log-file /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_basic_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 68-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:54:18 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63675262-dabd-4a2f-a363-f0045c90feba {"id":"63675262-dabd-4a2f-a363-f0045c90feba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149255} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce121c9c83 63675262-dabd-4a2f-a363-f0045c90feba /tidb/cdc/default/default/upstream/7375099854404710520 {"id":7375099854404710520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63675262-dabd-4a2f-a363-f0045c90feba {"id":"63675262-dabd-4a2f-a363-f0045c90feba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149255} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce121c9c83 63675262-dabd-4a2f-a363-f0045c90feba /tidb/cdc/default/default/upstream/7375099854404710520 {"id":7375099854404710520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/63675262-dabd-4a2f-a363-f0045c90feba {"id":"63675262-dabd-4a2f-a363-f0045c90feba","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149255} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce121c9c83 63675262-dabd-4a2f-a363-f0045c90feba /tidb/cdc/default/default/upstream/7375099854404710520 {"id":7375099854404710520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_basic_avro.cli.9945.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-simple-basic-avro-29424?protocol=simple&encoding-format=avro' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/conf/changefeed.toml Create changefeed successfully! ID: 6380e7a7-9093-499c-aa07-a0ca647666d7 Info: {"upstream_id":7375099854404710520,"namespace":"default","id":"6380e7a7-9093-499c-aa07-a0ca647666d7","sink_uri":"kafka://127.0.0.1:9092/ticdc-simple-basic-avro-29424?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-31T17:54:18.631378909+08:00","start_ts":450140375213932546,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140375213932546,"checkpoint_ts":450140375213932546,"checkpoint_time":"2024-05-31 17:54:18.476"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 69-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/mq_sink_dispatcher Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x table ddl_manager.finish_mark not exists for 70-th check, retry later table ddl_manager.finish_mark not exists for 71-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 72-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri May 31 17:54:25 CST 2024] <<<<<< START kafka consumer in kafka_simple_basic_avro case >>>>>> table ddl_manager.finish_mark not exists for 73-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3849d0200007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:12225, start at 2024-05-31 17:54:26.959329715 +0800 CST m=+1.755871420 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:26.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:26.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:26.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3849d0200007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:12225, start at 2024-05-31 17:54:26.959329715 +0800 CST m=+1.755871420 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:26.966 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:26.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:26.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3849d0d80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:12290, start at 2024-05-31 17:54:27.031531381 +0800 CST m=+1.775452714 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:27.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:26.998 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:26.998 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/mq_sink_dispatcher/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc) 3652131840 bytes in 20.33 secs (179604514 bytes/sec) [Pipeline] { [Pipeline] // container table ddl_manager.finish_mark not exists for 74-th check, retry later [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] sh [Fri May 31 17:54:29 CST 2024] <<<<<< START cdc server in mq_sink_dispatcher case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.1367413676.out server --log-file /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc.log --log-level info --data-dir /tmp/tidb_cdc_test/mq_sink_dispatcher/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] sh table test.finish_mark not exists for 2-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=f84eae1a-d4b9-411f-a2ae-142e58e384c3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-bxzm1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w pingcap_tiflow_pull_cdc_integration_kafka_test_2156-bxzm1 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory [Pipeline] sh =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G10 [Pipeline] sh Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=1443dae3-3dc5-4e41-8873-dadf02105261 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-wlvl3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-wlvl3 pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: kafka... <<================= \033[0;36m<<< Run all test success >>>\033[0m + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=e2927ad7-8b5a-444d-bb60-f555a62ae540 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-lxbz5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-lxbz5 pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh table ddl_manager.finish_mark not exists for 75-th check, retry later [Pipeline] container [Pipeline] { [Pipeline] timeout Timeout set to expire in 6 min 0 sec [Pipeline] { [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=c4cc95f2-2058-4631-bee4-f741f88fbdba BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-p2gjp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk pingcap_tiflow_pull_cdc_integration_kafka_test_2156-p2gjp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: kafka... <<================= table test.finish_mark not exists for 3-th check, retry later The 1 times to try to start tidb cluster... + echo Waiting for zookeeper to be ready... Waiting for zookeeper to be ready... + nc -z localhost 2181 + echo Waiting for kafka to be ready... Waiting for kafka to be ready... + nc -z localhost 9092 + echo Waiting for kafka-broker to be ready... Waiting for kafka-broker to be ready... + echo dump + nc localhost 2181 + grep brokers + awk {$1=$1;print} + grep -F -w /brokers/ids/1 /brokers/ids/1 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:54:32 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/292b7a18-8b62-4bc2-b003-90767acd3d0b {"id":"292b7a18-8b62-4bc2-b003-90767acd3d0b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149269} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1259417f 292b7a18-8b62-4bc2-b003-90767acd3d0b /tidb/cdc/default/default/upstream/7375099925415412390 {"id":7375099925415412390,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/292b7a18-8b62-4bc2-b003-90767acd3d0b {"id":"292b7a18-8b62-4bc2-b003-90767acd3d0b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149269} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1259417f 292b7a18-8b62-4bc2-b003-90767acd3d0b /tidb/cdc/default/default/upstream/7375099925415412390 {"id":7375099925415412390,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/292b7a18-8b62-4bc2-b003-90767acd3d0b {"id":"292b7a18-8b62-4bc2-b003-90767acd3d0b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149269} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1259417f 292b7a18-8b62-4bc2-b003-90767acd3d0b /tidb/cdc/default/default/upstream/7375099925415412390 {"id":7375099925415412390,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 76-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=d7cb8b81-55b4-45b7-899f-6a98142692bb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-lchjd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-lchjd pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.13761.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table test.finish_mark not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 77-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x + tso='450140379270873094 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140379270873094 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.13794.out cli changefeed create --start-ts=450140379270873094 '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/changefeed.toml Create changefeed successfully! ID: test Info: {"upstream_id":7375099925415412390,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-31T17:54:35.783721291+08:00","start_ts":450140379270873094,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["verify.t"],"partition":"index-value"},{"matcher":["dispatcher.index"],"partition":"index-value","index":"idx_a"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140379270873094,"checkpoint_ts":450140379270873094,"checkpoint_time":"2024-05-31 17:54:33.952"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 5-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140379270873094, "checkpoint_time": "2024-05-31 17:54:33.952", "error": null }' + echo '{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140379270873094, "checkpoint_time": "2024-05-31 17:54:33.952", "error": null }' { "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140379270873094, "checkpoint_time": "2024-05-31 17:54:33.952", "error": null } ++ echo '{' '"upstream_id":' 7375099925415412390, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450140379270873094, '"checkpoint_time":' '"2024-05-31' '17:54:33.952",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375099925415412390, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450140379270873094, '"checkpoint_time":' '"2024-05-31' '17:54:33.952",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table ddl_manager.finish_mark not exists for 78-th check, retry later Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/default_value Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table test.finish_mark not exists for 6-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140379270873094, "checkpoint_time": "2024-05-31 17:54:33.952", "error": null }' + echo '{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140379270873094, "checkpoint_time": "2024-05-31 17:54:33.952", "error": null }' { "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140379270873094, "checkpoint_time": "2024-05-31 17:54:33.952", "error": null } ++ echo '{' '"upstream_id":' 7375099925415412390, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450140379270873094, '"checkpoint_time":' '"2024-05-31' '17:54:33.952",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 1-th time, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 79-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384aa4e4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:1400, start at 2024-05-31 17:54:40.583427512 +0800 CST m=+1.867485317 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:40.591 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:40.569 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:40.569 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384aa4e4000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:1400, start at 2024-05-31 17:54:40.583427512 +0800 CST m=+1.867485317 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:40.591 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:40.569 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:40.569 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384aa5c40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:1478, start at 2024-05-31 17:54:40.673247825 +0800 CST m=+1.898203115 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:40.681 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:40.674 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:40.674 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark exists check diff successfully check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140380188639246, "checkpoint_time": "2024-05-31 17:54:37.453", "error": null }' + echo '{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140380188639246, "checkpoint_time": "2024-05-31 17:54:37.453", "error": null }' { "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140380188639246, "checkpoint_time": "2024-05-31 17:54:37.453", "error": null } ++ echo '{' '"upstream_id":' 7375099925415412390, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450140380188639246, '"checkpoint_time":' '"2024-05-31' '17:54:37.453",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \f\a\i\l\e\d ]] + echo 'changefeed state normal does not equal to failed' changefeed state normal does not equal to failed + exit 1 run task failed 2-th time, retry later [Pipeline] } wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ab9b0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:1351, start at 2024-05-31 17:54:41.917128189 +0800 CST m=+2.020410714 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:41.925 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:41.900 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:41.900 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ac3b80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:1289, start at 2024-05-31 17:54:42.542615076 +0800 CST m=+1.815703090 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:42.550 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:42.542 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:42.542 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ac3b80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:1289, start at 2024-05-31 17:54:42.542615076 +0800 CST m=+1.815703090 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:42.550 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:42.542 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:42.542 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] } [Fri May 31 17:54:43 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> wait process cdc.test exit for 4-th time... + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.28072809.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=4c6ff357-e792-490b-9ae9-259c73dd06e0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-js4zt GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-js4zt pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: kafka... <<================= [Pipeline] } wait process cdc.test exit for 5-th time... table test.finish_mark not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ad38c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2, pid:1278, start at 2024-05-31 17:54:43.601963712 +0800 CST m=+1.959453343 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:43.611 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:43.606 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:43.606 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ad38c001a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2, pid:1278, start at 2024-05-31 17:54:43.601963712 +0800 CST m=+1.959453343 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:43.611 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:43.606 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:43.606 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) wait process cdc.test exit for 6-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ab9b0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:1351, start at 2024-05-31 17:54:41.917128189 +0800 CST m=+2.020410714 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:41.925 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:41.900 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:41.900 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ab828000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:1420, start at 2024-05-31 17:54:41.817030816 +0800 CST m=+1.861107096 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:41.824 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:41.802 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:41.802 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 7-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384adc180018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:1343, start at 2024-05-31 17:54:44.135817872 +0800 CST m=+1.741903486 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:44.143 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:44.102 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:44.102 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384adc180018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:1343, start at 2024-05-31 17:54:44.135817872 +0800 CST m=+1.741903486 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:44.143 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:44.102 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:44.102 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ae0a40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:1423, start at 2024-05-31 17:54:44.410264025 +0800 CST m=+1.946052772 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:44.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:44.393 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:44.393 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/default_value/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/default_value/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/default_value/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/default_value/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 8-th time... process cdc.test already exit [Fri May 31 17:54:45 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ac7a40003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:1357, start at 2024-05-31 17:54:42.793888055 +0800 CST m=+2.002780943 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:42.801 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:42.793 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:42.793 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } check_changefeed_state http://127.0.0.1:2379 test failed ErrDispatcherFailed + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=failed + error_msg=ErrDispatcherFailed + tls_dir=ErrDispatcherFailed + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 450140380319449124, "checkpoint_time": "2024-05-31 17:54:37.952", "error": { "time": "2024-05-31T17:54:42.523793036+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } }' + echo '{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 450140380319449124, "checkpoint_time": "2024-05-31 17:54:37.952", "error": { "time": "2024-05-31T17:54:42.523793036+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } }' { "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "failed", "checkpoint_tso": 450140380319449124, "checkpoint_time": "2024-05-31 17:54:37.952", "error": { "time": "2024-05-31T17:54:42.523793036+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrDispatcherFailed", "message": "[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375099925415412390, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 450140380319449124, '"checkpoint_time":' '"2024-05-31' '17:54:37.952",' '"error":' '{' '"time":' '"2024-05-31T17:54:42.523793036+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: index, index: 'idx_a"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375099925415412390, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"failed",' '"checkpoint_tso":' 450140380319449124, '"checkpoint_time":' '"2024-05-31' '17:54:37.952",' '"error":' '{' '"time":' '"2024-05-31T17:54:42.523793036+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrDispatcherFailed",' '"message":' '"[CDC:ErrDispatcherFailed]index' not found when dispatch event, table: index, index: 'idx_a"' '}' '}' + message='[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a' + [[ ! [CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a =~ ErrDispatcherFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14000.out cli changefeed update -c test '--sink-uri=kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/conf/new_changefeed.toml --no-confirm Diff of changefeed config: {Type:update Path:[Config SyncPointInterval] From: To:0xc0035d32e8} {Type:update Path:[Config SyncPointRetention] From: To:0xc0035d32f8} {Type:update Path:[Config Sink DispatchRules 0 Matcher 0] From:verify.t To:dispatcher.index} {Type:delete Path:[Config Sink DispatchRules 1 Matcher 0] From:dispatcher.index To:} {Type:delete Path:[Config Sink DispatchRules 1 PartitionRule] From:index-value To:} {Type:delete Path:[Config Sink DispatchRules 1 IndexName] From:idx_a To:} {Type:update Path:[Config Consistent] From: To:0xc0011b5d50} Update changefeed config successfully! ID: test Info: {"upstream_id":7375099925415412390,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/dispatcher-test?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-31T17:54:35.783721291+08:00","start_ts":450140379270873094,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","dispatchers":[{"matcher":["dispatcher.index"],"partition":"index-value"}],"encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"failed","error":{"addr":"127.0.0.1:8300","code":"CDC:ErrDispatcherFailed","message":"[CDC:ErrDispatcherFailed]index not found when dispatch event, table: index, index: idx_a"},"creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":0,"checkpoint_ts":450140380319449124,"checkpoint_time":"2024-05-31 17:54:37.952"} PASS [Pipeline] // cache [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ad5e80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2, pid:1356, start at 2024-05-31 17:54:43.735062323 +0800 CST m=+2.026334509 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:43.743 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:43.706 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:43.706 +0800 All versions after safe point can be accessed. (DO NOT EDIT) coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // timeout [Pipeline] } table test.finish_mark not exists for 10-th check, retry later The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:54:46 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e6e06dfd-84f7-487e-8ed8-a39ea81f7110 {"id":"e6e06dfd-84f7-487e-8ed8-a39ea81f7110","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149283} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1290027e e6e06dfd-84f7-487e-8ed8-a39ea81f7110 /tidb/cdc/default/default/upstream/7375099982499073988 {"id":7375099982499073988,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e6e06dfd-84f7-487e-8ed8-a39ea81f7110 {"id":"e6e06dfd-84f7-487e-8ed8-a39ea81f7110","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149283} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1290027e e6e06dfd-84f7-487e-8ed8-a39ea81f7110 /tidb/cdc/default/default/upstream/7375099982499073988 {"id":7375099982499073988,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e6e06dfd-84f7-487e-8ed8-a39ea81f7110 {"id":"e6e06dfd-84f7-487e-8ed8-a39ea81f7110","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149283} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1290027e e6e06dfd-84f7-487e-8ed8-a39ea81f7110 /tidb/cdc/default/default/upstream/7375099982499073988 {"id":7375099982499073988,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:54:46 CST 2024] <<<<<< START kafka consumer in gc_safepoint case >>>>>> 0 [Pipeline] // timeout [Pipeline] // timeout check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2853.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2802.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] // cache [Pipeline] } + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.mq_sink_dispatcher.cli.14021.out cli changefeed resume -c test [Pipeline] // timeout [Pipeline] } [Fri May 31 17:54:47 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.29182920.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // dir [Pipeline] } PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [Pipeline] // container [Pipeline] // container [Pipeline] sh [Fri May 31 17:54:47 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.28002802.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] sh table test.finish_mark not exists for 11-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes + set +x + tso='450140382692114433 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140382692114433 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:54:48 CST 2024] <<<<<< START cdc server in default_value case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.28842886.out server --log-file /tmp/tidb_cdc_test/default_value/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/default_value/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=1df7d14a-4594-43e5-a37b-210653fe7336 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-qppb5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-qppb5 pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: kafka... <<================= + set +x + tso='450140382780981249 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140382780981249 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "readproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.db"="sink_retry" "threadcount"="2" "requestdistribution"="uniform" "scanproportion"="0" "mysql.user"="root" "readallfields"="true" "insertproportion"="0" "updateproportion"="0" "mysql.port"="4000" "recordcount"="10" "operationcount"="0" "workload"="core" ********************************************** Run finished, takes 9.443074ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1807.9, Avg(us): 1811, Min(us): 1093, Max(us): 3816, 95th(us): 4000, 99th(us): 4000 [Fri May 31 17:54:48 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.28562858.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check_changefeed_state http://127.0.0.1:2379 test normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=test + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test -s + info='{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140380319449124, "checkpoint_time": "2024-05-31 17:54:37.952", "error": null }' + echo '{ "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140380319449124, "checkpoint_time": "2024-05-31 17:54:37.952", "error": null }' { "upstream_id": 7375099925415412390, "namespace": "default", "id": "test", "state": "normal", "checkpoint_tso": 450140380319449124, "checkpoint_time": "2024-05-31 17:54:37.952", "error": null } ++ echo '{' '"upstream_id":' 7375099925415412390, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450140380319449124, '"checkpoint_time":' '"2024-05-31' '17:54:37.952",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375099925415412390, '"namespace":' '"default",' '"id":' '"test",' '"state":' '"normal",' '"checkpoint_tso":' 450140380319449124, '"checkpoint_time":' '"2024-05-31' '17:54:37.952",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff failed 2-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=1ad99469-3201-4050-b4ad-4303d7f9a58a BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fv4zk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fv4zk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 12-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [Pipeline] sh + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:54:50 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2c2efb56-f79f-422e-b660-3ea9dc30ace8 {"id":"2c2efb56-f79f-422e-b660-3ea9dc30ace8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149287} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce128ec50f 2c2efb56-f79f-422e-b660-3ea9dc30ace8 /tidb/cdc/default/default/upstream/7375099982795776821 {"id":7375099982795776821,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2c2efb56-f79f-422e-b660-3ea9dc30ace8 {"id":"2c2efb56-f79f-422e-b660-3ea9dc30ace8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149287} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce128ec50f 2c2efb56-f79f-422e-b660-3ea9dc30ace8 /tidb/cdc/default/default/upstream/7375099982795776821 {"id":7375099982795776821,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2c2efb56-f79f-422e-b660-3ea9dc30ace8 {"id":"2c2efb56-f79f-422e-b660-3ea9dc30ace8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149287} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce128ec50f 2c2efb56-f79f-422e-b660-3ea9dc30ace8 /tidb/cdc/default/default/upstream/7375099982795776821 {"id":7375099982795776821,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:54:50 CST 2024] <<<<<< START kafka consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 3e1db845-716e-4b01-a899-f486e06c83a9 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=3e1db845-716e-4b01-a899-f486e06c83a9 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 3e1db845-716e-4b01-a899-f486e06c83a9 -s + info='{ "upstream_id": 7375099982795776821, "namespace": "default", "id": "3e1db845-716e-4b01-a899-f486e06c83a9", "state": "normal", "checkpoint_tso": 450140383608832001, "checkpoint_time": "2024-05-31 17:54:50.500", "error": null }' + echo '{ "upstream_id": 7375099982795776821, "namespace": "default", "id": "3e1db845-716e-4b01-a899-f486e06c83a9", "state": "normal", "checkpoint_tso": 450140383608832001, "checkpoint_time": "2024-05-31 17:54:50.500", "error": null }' { "upstream_id": 7375099982795776821, "namespace": "default", "id": "3e1db845-716e-4b01-a899-f486e06c83a9", "state": "normal", "checkpoint_tso": 450140383608832001, "checkpoint_time": "2024-05-31 17:54:50.500", "error": null } ++ echo '{' '"upstream_id":' 7375099982795776821, '"namespace":' '"default",' '"id":' '"3e1db845-716e-4b01-a899-f486e06c83a9",' '"state":' '"normal",' '"checkpoint_tso":' 450140383608832001, '"checkpoint_time":' '"2024-05-31' '17:54:50.500",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375099982795776821, '"namespace":' '"default",' '"id":' '"3e1db845-716e-4b01-a899-f486e06c83a9",' '"state":' '"normal",' '"checkpoint_tso":' 450140383608832001, '"checkpoint_time":' '"2024-05-31' '17:54:50.500",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff successfully check_safepoint_forward http://127.0.0.1:2379 7375099982499073988 450140383417466880 450140382486855683 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:54:51 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaf54bea-0b25-4fef-9b55-6210b8174327 {"id":"aaf54bea-0b25-4fef-9b55-6210b8174327","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce129b488b aaf54bea-0b25-4fef-9b55-6210b8174327 /tidb/cdc/default/default/upstream/7375099990813253095 {"id":7375099990813253095,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaf54bea-0b25-4fef-9b55-6210b8174327 {"id":"aaf54bea-0b25-4fef-9b55-6210b8174327","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce129b488b aaf54bea-0b25-4fef-9b55-6210b8174327 /tidb/cdc/default/default/upstream/7375099990813253095 {"id":7375099990813253095,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaf54bea-0b25-4fef-9b55-6210b8174327 {"id":"aaf54bea-0b25-4fef-9b55-6210b8174327","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce129b488b aaf54bea-0b25-4fef-9b55-6210b8174327 /tidb/cdc/default/default/upstream/7375099990813253095 {"id":7375099990813253095,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Pipeline] // container [Pipeline] // container [Pipeline] sh check diff failed 1-th time, retry later [Fri May 31 17:54:51 CST 2024] <<<<<< START kafka consumer in kv_client_stream_reconnect case >>>>>> [Pipeline] sh table test.finish_mark exists check diff successfully + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G15 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:54:51 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2219319-bdb5-4641-a569-1343c0f910b4 {"id":"b2219319-bdb5-4641-a569-1343c0f910b4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce129c7188 b2219319-bdb5-4641-a569-1343c0f910b4 /tidb/cdc/default/default/upstream/7375099999023968370 {"id":7375099999023968370,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2219319-bdb5-4641-a569-1343c0f910b4 {"id":"b2219319-bdb5-4641-a569-1343c0f910b4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce129c7188 b2219319-bdb5-4641-a569-1343c0f910b4 /tidb/cdc/default/default/upstream/7375099999023968370 {"id":7375099999023968370,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2219319-bdb5-4641-a569-1343c0f910b4 {"id":"b2219319-bdb5-4641-a569-1343c0f910b4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149288} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce129c7188 b2219319-bdb5-4641-a569-1343c0f910b4 /tidb/cdc/default/default/upstream/7375099999023968370 {"id":7375099999023968370,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.default_value.cli.2934.out cli changefeed create --start-ts=450140382692114433 '--sink-uri=kafka://127.0.0.1:9092/ticdc-default-value-test-28815?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 6fdef03e-72a8-4b18-89c8-6771652707dc Info: {"upstream_id":7375099999023968370,"namespace":"default","id":"6fdef03e-72a8-4b18-89c8-6771652707dc","sink_uri":"kafka://127.0.0.1:9092/ticdc-default-value-test-28815?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:54:52.029743285+08:00","start_ts":450140382692114433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140382692114433,"checkpoint_ts":450140382692114433,"checkpoint_time":"2024-05-31 17:54:47.003"} PASS [Pipeline] // dir [Pipeline] } table test.finish_mark not exists for 13-th check, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 d58a79b8-2d2b-47e5-a32a-0d7f03246cda stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=d58a79b8-2d2b-47e5-a32a-0d7f03246cda + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c d58a79b8-2d2b-47e5-a32a-0d7f03246cda -s [Pipeline] // container + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G16 [Pipeline] sh Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=cc174c2d-0bba-409c-b8b4-827112b66b88 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-tr2x6 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1 pingcap_tiflow_pull_cdc_integration_kafka_test_2156-tr2x6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... coverage: 2.5% of statements in github.com/pingcap/tiflow/... + info='{ "upstream_id": 7375099982499073988, "namespace": "default", "id": "d58a79b8-2d2b-47e5-a32a-0d7f03246cda", "state": "stopped", "checkpoint_tso": 450140383941492738, "checkpoint_time": "2024-05-31 17:54:51.769", "error": null }' + echo '{ "upstream_id": 7375099982499073988, "namespace": "default", "id": "d58a79b8-2d2b-47e5-a32a-0d7f03246cda", "state": "stopped", "checkpoint_tso": 450140383941492738, "checkpoint_time": "2024-05-31 17:54:51.769", "error": null }' { "upstream_id": 7375099982499073988, "namespace": "default", "id": "d58a79b8-2d2b-47e5-a32a-0d7f03246cda", "state": "stopped", "checkpoint_tso": 450140383941492738, "checkpoint_time": "2024-05-31 17:54:51.769", "error": null } ++ echo '{' '"upstream_id":' 7375099982499073988, '"namespace":' '"default",' '"id":' '"d58a79b8-2d2b-47e5-a32a-0d7f03246cda",' '"state":' '"stopped",' '"checkpoint_tso":' 450140383941492738, '"checkpoint_time":' '"2024-05-31' '17:54:51.769",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7375099982499073988, '"namespace":' '"default",' '"id":' '"d58a79b8-2d2b-47e5-a32a-0d7f03246cda",' '"state":' '"stopped",' '"checkpoint_tso":' 450140383941492738, '"checkpoint_time":' '"2024-05-31' '17:54:51.769",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7375099982499073988 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:54:51 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d1717ac-7652-4a3f-ba13-0df7d4760d68 {"id":"7d1717ac-7652-4a3f-ba13-0df7d4760d68","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1293d0ad 7d1717ac-7652-4a3f-ba13-0df7d4760d68 /tidb/cdc/default/default/upstream/7375099983345546229 {"id":7375099983345546229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d1717ac-7652-4a3f-ba13-0df7d4760d68 {"id":"7d1717ac-7652-4a3f-ba13-0df7d4760d68","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1293d0ad 7d1717ac-7652-4a3f-ba13-0df7d4760d68 /tidb/cdc/default/default/upstream/7375099983345546229 {"id":7375099983345546229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d1717ac-7652-4a3f-ba13-0df7d4760d68 {"id":"7d1717ac-7652-4a3f-ba13-0df7d4760d68","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1293d0ad 7d1717ac-7652-4a3f-ba13-0df7d4760d68 /tidb/cdc/default/default/upstream/7375099983345546229 {"id":7375099983345546229,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2916.out cli changefeed create --start-ts=450140382780981249 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-3647?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: f9ea6d88-3d1c-45ab-be4b-5ba0a9baec6b Info: {"upstream_id":7375099983345546229,"namespace":"default","id":"f9ea6d88-3d1c-45ab-be4b-5ba0a9baec6b","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-3647?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:54:52.414580285+08:00","start_ts":450140382780981249,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140382780981249,"checkpoint_ts":450140382780981249,"checkpoint_time":"2024-05-31 17:54:47.342"} PASS Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=0f81de00-9be0-45b5-af25-7e9c13dbe387 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fw18n-j7mdr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fw18n GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-fw18n pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fw18n-j7mdr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fw18n-j7mdr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:54:52 CST 2024] <<<<<< run test case owner_resign success! >>>>>> cdc.test: no process found wait process cdc.test exit for 1-th time... process cdc.test already exit [Fri May 31 17:54:52 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> [Pipeline] // withCredentials [Pipeline] } coverage: 2.5% of statements in github.com/pingcap/tiflow/... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=f376a5c0-c90f-4fb7-b232-a0ba661725a3 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-z6b6n-l09dl HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-z6b6n GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-z6b6n pingcap-tiflow-pull-cdc-integration-kafka-test-2156-z6b6n-l09dl GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-z6b6n-l09dl GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:54:52 CST 2024] <<<<<< skip test case clustered_index for kafka! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:54:52 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> [Pipeline] // withCredentials [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x [Fri May 31 17:54:53 CST 2024] <<<<<< START kafka consumer in default_value case >>>>>> go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240520031014-397a460dd067 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/grpc v1.63.2 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240520031014-397a460dd067 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/multierr v1.11.0 go: downloading golang.org/x/sys v0.20.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda check diff failed 2-th time, retry later [Pipeline] // timeout [Pipeline] } table test.finish_mark not exists for 14-th check, retry later + set +x [Fri May 31 17:54:53 CST 2024] <<<<<< START kafka consumer in sink_retry case >>>>>> [Pipeline] // timeout + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh kafka G14 [Pipeline] } [Pipeline] // stage [Pipeline] } Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=ac4e637b-bee3-4029-8dc4-99ce77c9dc20 JENKINS_NODE_COOKIE=1e764433-25c7-4cab-b320-d174db9a10db BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/ GOLANG_VERSION=1.21.0 HOSTNAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_kafka_test","buildid":"1796475898619236354","prowjobid":"ac4e637b-bee3-4029-8dc4-99ce77c9dc20","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"350a2fb4596b489af648982082e60ea5735178ff","base_link":"https://github.com/pingcap/tiflow/commit/350a2fb4596b489af648982082e60ea5735178ff","pulls":[{"number":11081,"author":"3AceShowHand","sha":"f59703ae3c583e094370068a597537791092805f","title":"mounter(ticdc): optimize mounter cpu usage by reduce memory allocation on map and slice","link":"https://github.com/pingcap/tiflow/pull/11081","commit_link":"https://github.com/pingcap/tiflow/pull/11081/commits/f59703ae3c583e094370068a597537791092805f","author_link":"https://github.com/3AceShowHand"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_kafka_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#2156 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1796475898619236354 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742 JOB_BASE_NAME=pull_cdc_integration_kafka_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=e64357d6ddd9be700fd5c8d87859c826b78da53a RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/2156/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_kafka_test_2156-9jk86 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp TINI_VERSION=v0.19.0 CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-8f1d433d6527e85b4c28b432b07fa0c56dd3090c9176381d29a5b4531676247a NODE_LABELS=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn pingcap_tiflow_pull_cdc_integration_kafka_test_2156-9jk86 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_kafka_test/display/redirect BUILD_NUMBER=2156 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // stage [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // container [Pipeline] } [Pipeline] // container [Pipeline] } run task successfully [Pipeline] // withEnv [Pipeline] } check diff successfully go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/text v0.15.0 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240516031922-38e0dca30c8c go: downloading github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading golang.org/x/net v0.25.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading golang.org/x/oauth2 v0.20.0 [Pipeline] // withEnv table test.finish_mark not exists for 15-th check, retry later check_changefeed_state http://127.0.0.1:2379 d58a79b8-2d2b-47e5-a32a-0d7f03246cda normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=d58a79b8-2d2b-47e5-a32a-0d7f03246cda + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c d58a79b8-2d2b-47e5-a32a-0d7f03246cda -s + info='{ "upstream_id": 7375099982499073988, "namespace": "default", "id": "d58a79b8-2d2b-47e5-a32a-0d7f03246cda", "state": "normal", "checkpoint_tso": 450140383941492738, "checkpoint_time": "2024-05-31 17:54:51.769", "error": null }' + echo '{ "upstream_id": 7375099982499073988, "namespace": "default", "id": "d58a79b8-2d2b-47e5-a32a-0d7f03246cda", "state": "normal", "checkpoint_tso": 450140383941492738, "checkpoint_time": "2024-05-31 17:54:51.769", "error": null }' { "upstream_id": 7375099982499073988, "namespace": "default", "id": "d58a79b8-2d2b-47e5-a32a-0d7f03246cda", "state": "normal", "checkpoint_tso": 450140383941492738, "checkpoint_time": "2024-05-31 17:54:51.769", "error": null } ++ echo '{' '"upstream_id":' 7375099982499073988, '"namespace":' '"default",' '"id":' '"d58a79b8-2d2b-47e5-a32a-0d7f03246cda",' '"state":' '"normal",' '"checkpoint_tso":' 450140383941492738, '"checkpoint_time":' '"2024-05-31' '17:54:51.769",' '"error":' null '}' ++ jq -r .state [Pipeline] } wait process cdc.test exit for 1-th time... go: downloading google.golang.org/api v0.170.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 [Pipeline] // node [Pipeline] } + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375099982499073988, '"namespace":' '"default",' '"id":' '"d58a79b8-2d2b-47e5-a32a-0d7f03246cda",' '"state":' '"normal",' '"checkpoint_tso":' 450140383941492738, '"checkpoint_time":' '"2024-05-31' '17:54:51.769",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7375099982499073988 450140383941492737 450140383941492738 Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 [Pipeline] // node [Pipeline] } Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/new_ci_collation Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [Pipeline] // podTemplate go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 wait process cdc.test exit for 2-th time... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:54:56 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> [Pipeline] } [Pipeline] // podTemplate [Pipeline] } The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] // withEnv [Pipeline] } run task successfully [Pipeline] // withEnv =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: kafka... <<================= cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:54:57 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> [Pipeline] } [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) check_changefeed_state http://127.0.0.1:2379 d58a79b8-2d2b-47e5-a32a-0d7f03246cda stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=d58a79b8-2d2b-47e5-a32a-0d7f03246cda + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c d58a79b8-2d2b-47e5-a32a-0d7f03246cda -s Verifying downstream PD is started... [Pipeline] // stage [Pipeline] } [Pipeline] // stage + info='{ "upstream_id": 7375099982499073988, "namespace": "default", "id": "d58a79b8-2d2b-47e5-a32a-0d7f03246cda", "state": "stopped", "checkpoint_tso": 450140383941492738, "checkpoint_time": "2024-05-31 17:54:51.769", "error": null }' + echo '{ "upstream_id": 7375099982499073988, "namespace": "default", "id": "d58a79b8-2d2b-47e5-a32a-0d7f03246cda", "state": "stopped", "checkpoint_tso": 450140383941492738, "checkpoint_time": "2024-05-31 17:54:51.769", "error": null }' { "upstream_id": 7375099982499073988, "namespace": "default", "id": "d58a79b8-2d2b-47e5-a32a-0d7f03246cda", "state": "stopped", "checkpoint_tso": 450140383941492738, "checkpoint_time": "2024-05-31 17:54:51.769", "error": null } ++ echo '{' '"upstream_id":' 7375099982499073988, '"namespace":' '"default",' '"id":' '"d58a79b8-2d2b-47e5-a32a-0d7f03246cda",' '"state":' '"stopped",' '"checkpoint_tso":' 450140383941492738, '"checkpoint_time":' '"2024-05-31' '17:54:51.769",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7375099982499073988, '"namespace":' '"default",' '"id":' '"d58a79b8-2d2b-47e5-a32a-0d7f03246cda",' '"state":' '"stopped",' '"checkpoint_tso":' 450140383941492738, '"checkpoint_time":' '"2024-05-31' '17:54:51.769",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully [Pipeline] // cache [Pipeline] } [Pipeline] } check_changefeed_state http://127.0.0.1:2379 87d2f850-14a6-47a6-a06b-aa9f235e1599 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=87d2f850-14a6-47a6-a06b-aa9f235e1599 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 87d2f850-14a6-47a6-a06b-aa9f235e1599 -s [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } + info='{ "upstream_id": 7375099982499073988, "namespace": "default", "id": "87d2f850-14a6-47a6-a06b-aa9f235e1599", "state": "normal", "checkpoint_tso": 450140385488404484, "checkpoint_time": "2024-05-31 17:54:57.670", "error": null }' + echo '{ "upstream_id": 7375099982499073988, "namespace": "default", "id": "87d2f850-14a6-47a6-a06b-aa9f235e1599", "state": "normal", "checkpoint_tso": 450140385488404484, "checkpoint_time": "2024-05-31 17:54:57.670", "error": null }' { "upstream_id": 7375099982499073988, "namespace": "default", "id": "87d2f850-14a6-47a6-a06b-aa9f235e1599", "state": "normal", "checkpoint_tso": 450140385488404484, "checkpoint_time": "2024-05-31 17:54:57.670", "error": null } ++ echo '{' '"upstream_id":' 7375099982499073988, '"namespace":' '"default",' '"id":' '"87d2f850-14a6-47a6-a06b-aa9f235e1599",' '"state":' '"normal",' '"checkpoint_tso":' 450140385488404484, '"checkpoint_time":' '"2024-05-31' '17:54:57.670",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375099982499073988, '"namespace":' '"default",' '"id":' '"87d2f850-14a6-47a6-a06b-aa9f235e1599",' '"state":' '"normal",' '"checkpoint_tso":' 450140385488404484, '"checkpoint_time":' '"2024-05-31' '17:54:57.670",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7375099982499073988 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384bb5600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:1353, start at 2024-05-31 17:54:58.058249073 +0800 CST m=+1.853135650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:58.065 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:58.058 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:58.058 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384bb5600014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:1353, start at 2024-05-31 17:54:58.058249073 +0800 CST m=+1.853135650 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:58.065 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:58.058 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:58.058 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384bb6dc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:1425, start at 2024-05-31 17:54:58.143897918 +0800 CST m=+1.877319426 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:56:58.150 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:54:58.153 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:44:58.153 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Pipeline] // timeout table test.finish_mark not exists for 16-th check, retry later [Pipeline] } Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // stage [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 start tidb cluster in /tmp/tidb_cdc_test/many_pk_or_uk Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table test.finish_mark not exists for 17-th check, retry later The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2865.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: kafka... <<================= [Fri May 31 17:55:00 CST 2024] <<<<<< run test case sink_hang success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384bdd480007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b, pid:1395, start at 2024-05-31 17:55:00.569601213 +0800 CST m=+1.908749447 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:00.576 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:00.562 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:00.562 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384bdd480007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b, pid:1395, start at 2024-05-31 17:55:00.569601213 +0800 CST m=+1.908749447 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:00.576 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:00.562 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:00.562 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384bde140005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b, pid:1470, start at 2024-05-31 17:55:00.619433477 +0800 CST m=+1.888805972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:00.626 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:00.613 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:00.613 +0800 All versions after safe point can be accessed. (DO NOT EDIT) run task successfully Verifying downstream PD is started... Changefeed remove successfully. ID: d58a79b8-2d2b-47e5-a32a-0d7f03246cda CheckpointTs: 450140383941492738 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-26791?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_forward http://127.0.0.1:2379 7375099982499073988 450140385488404483 450140385488404484 450140383941492738 Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:54:58 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark exists check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='450140386311274497 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140386311274497 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:55:02 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.29002902.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 run task successfully Changefeed remove successfully. ID: 87d2f850-14a6-47a6-a06b-aa9f235e1599 CheckpointTs: 450140386562932739 SinkURI: kafka://127.0.0.1:9092/ticdc-gc-safepoint-26791?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 check_safepoint_cleared http://127.0.0.1:2379 7375099982499073988 run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c08dc001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:1345, start at 2024-05-31 17:55:03.380970491 +0800 CST m=+1.989751840 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:03.389 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:03.351 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:03.351 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c08dc001e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:1345, start at 2024-05-31 17:55:03.380970491 +0800 CST m=+1.989751840 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:03.389 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:03.351 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:03.351 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c09980014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:1426, start at 2024-05-31 17:55:03.419199456 +0800 CST m=+1.962962881 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:03.427 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:03.398 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:03.398 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/new_ci_collation/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/new_ci_collation/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh: line 1: 14100 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/new_changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:55:04 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2286.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:05 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed358f12-d584-49ec-96bf-8f0a91129d2f {"id":"ed358f12-d584-49ec-96bf-8f0a91129d2f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149302} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12ce7588 ed358f12-d584-49ec-96bf-8f0a91129d2f /tidb/cdc/default/default/upstream/7375100056648633065 {"id":7375100056648633065,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed358f12-d584-49ec-96bf-8f0a91129d2f {"id":"ed358f12-d584-49ec-96bf-8f0a91129d2f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149302} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12ce7588 ed358f12-d584-49ec-96bf-8f0a91129d2f /tidb/cdc/default/default/upstream/7375100056648633065 {"id":7375100056648633065,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed358f12-d584-49ec-96bf-8f0a91129d2f {"id":"ed358f12-d584-49ec-96bf-8f0a91129d2f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149302} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12ce7588 ed358f12-d584-49ec-96bf-8f0a91129d2f /tidb/cdc/default/default/upstream/7375100056648633065 {"id":7375100056648633065,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2951.out cli changefeed create --start-ts=450140386311274497 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resolve-lock-test-4353?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Create changefeed successfully! ID: 958fa870-819c-4ab8-8e5b-881a547ce655 Info: {"upstream_id":7375100056648633065,"namespace":"default","id":"958fa870-819c-4ab8-8e5b-881a547ce655","sink_uri":"kafka://127.0.0.1:9092/ticdc-resolve-lock-test-4353?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:55:05.73189156+08:00","start_ts":450140386311274497,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140386311274497,"checkpoint_ts":450140386311274497,"checkpoint_time":"2024-05-31 17:55:00.809"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Fri May 31 17:55:05 CST 2024] <<<<<< START cdc server in new_ci_collation case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.new_ci_collation.28322834.out server --log-file /tmp/tidb_cdc_test/new_ci_collation/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/new_ci_collation/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } table test.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container + set +x + tso='450140387399958529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140387399958529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:55:06 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.28262828.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } + set +x [Fri May 31 17:55:07 CST 2024] <<<<<< START kafka consumer in resolve_lock case >>>>>> go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240516031922-38e0dca30c8c go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240520031014-397a460dd067 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240520031014-397a460dd067 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/google/btree v1.1.2 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading google.golang.org/grpc v1.63.2 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/docker/go-units v0.5.0 go: downloading google.golang.org/protobuf v1.33.0 [Pipeline] // stage start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector Starting Upstream PD... [Pipeline] } go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/sys v0.20.0 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/cloudfoundry/gosigar v1.3.6 Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... go: downloading golang.org/x/net v0.25.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading golang.org/x/text v0.15.0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c4c440005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k, pid:1485, start at 2024-05-31 17:55:07.66849669 +0800 CST m=+1.763773995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:07.675 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:07.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:07.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c4c440005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k, pid:1485, start at 2024-05-31 17:55:07.66849669 +0800 CST m=+1.763773995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:07.675 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:07.665 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:07.665 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c4dc40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k, pid:1562, start at 2024-05-31 17:55:07.7993185 +0800 CST m=+1.821881736 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:07.806 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:07.761 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:07.761 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/many_pk_or_uk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c5018000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn, pid:1326, start at 2024-05-31 17:55:07.935084435 +0800 CST m=+3.830519191 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:07.949 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:07.959 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:07.959 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c5018000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn, pid:1326, start at 2024-05-31 17:55:07.935084435 +0800 CST m=+3.830519191 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:07.949 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:07.959 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:07.959 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c4ffc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn, pid:1363, start at 2024-05-31 17:55:07.94748436 +0800 CST m=+3.750913239 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:07.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:07.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:07.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:08 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98e88d35-dd4d-4fbf-8622-a7b6261f6168 {"id":"98e88d35-dd4d-4fbf-8622-a7b6261f6168","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149306} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12e48382 98e88d35-dd4d-4fbf-8622-a7b6261f6168 /tidb/cdc/default/default/upstream/7375100072573295011 {"id":7375100072573295011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98e88d35-dd4d-4fbf-8622-a7b6261f6168 {"id":"98e88d35-dd4d-4fbf-8622-a7b6261f6168","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149306} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12e48382 98e88d35-dd4d-4fbf-8622-a7b6261f6168 /tidb/cdc/default/default/upstream/7375100072573295011 {"id":7375100072573295011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/98e88d35-dd4d-4fbf-8622-a7b6261f6168 {"id":"98e88d35-dd4d-4fbf-8622-a7b6261f6168","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149306} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12e48382 98e88d35-dd4d-4fbf-8622-a7b6261f6168 /tidb/cdc/default/default/upstream/7375100072573295011 {"id":7375100072573295011,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: c9aefc69-309e-4317-8c33-761fe73035fe Info: {"upstream_id":7375100072573295011,"namespace":"default","id":"c9aefc69-309e-4317-8c33-761fe73035fe","sink_uri":"kafka://127.0.0.1:9092/ticdc-new_ci_collation-test-24632?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:55:09.143357786+08:00","start_ts":450140387606790145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140387606790145,"checkpoint_ts":450140387606790145,"checkpoint_time":"2024-05-31 17:55:05.751"} [Fri May 31 17:55:09 CST 2024] <<<<<< START kafka consumer in new_ci_collation case >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 table new_ci_collation_test.t1 not exists for 1-th check, retry later go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading golang.org/x/tools v0.21.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading golang.org/x/oauth2 v0.20.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.20.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:09 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b05576e-adc7-4f72-92c6-e45dc8cdce51 {"id":"6b05576e-adc7-4f72-92c6-e45dc8cdce51","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149306} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12da0294 6b05576e-adc7-4f72-92c6-e45dc8cdce51 /tidb/cdc/default/default/upstream/7375100068325555738 {"id":7375100068325555738,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b05576e-adc7-4f72-92c6-e45dc8cdce51 {"id":"6b05576e-adc7-4f72-92c6-e45dc8cdce51","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149306} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12da0294 6b05576e-adc7-4f72-92c6-e45dc8cdce51 /tidb/cdc/default/default/upstream/7375100068325555738 {"id":7375100068325555738,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b05576e-adc7-4f72-92c6-e45dc8cdce51 {"id":"6b05576e-adc7-4f72-92c6-e45dc8cdce51","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149306} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12da0294 6b05576e-adc7-4f72-92c6-e45dc8cdce51 /tidb/cdc/default/default/upstream/7375100068325555738 {"id":7375100068325555738,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 94e007ab-1e36-431c-95a5-1e12ded8abd9 Info: {"upstream_id":7375100068325555738,"namespace":"default","id":"94e007ab-1e36-431c-95a5-1e12ded8abd9","sink_uri":"kafka://127.0.0.1:9092/ticdc-tiflash-test-32315?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:55:09.705387666+08:00","start_ts":450140387399958529,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140387399958529,"checkpoint_ts":450140387399958529,"checkpoint_time":"2024-05-31 17:55:04.962"} [Fri May 31 17:55:09 CST 2024] <<<<<< START kafka consumer in tiflash case >>>>>> table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2872.out cli tso query --pd=http://127.0.0.1:2379 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b table test.finish_mark not exists for 4-th check, retry later [Fri May 31 17:55:10 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.26622664.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table new_ci_collation_test.t1 exists table new_ci_collation_test.t2 not exists for 1-th check, retry later table cdc_tiflash_test.multi_data_type not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450140388868751361 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140388868751361 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:55:11 CST 2024] <<<<<< START cdc server in many_pk_or_uk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.29042906.out server --log-file /tmp/tidb_cdc_test/many_pk_or_uk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/many_pk_or_uk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c6ad8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:7393, start at 2024-05-31 17:55:09.638703217 +0800 CST m=+1.957189192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:09.646 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:09.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:09.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c6ad8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:7393, start at 2024-05-31 17:55:09.638703217 +0800 CST m=+1.957189192 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:09.646 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:09.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:09.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384c6c6c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:7460, start at 2024-05-31 17:55:09.732563401 +0800 CST m=+1.988324842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:09.740 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:09.723 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:09.723 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 5-th check, retry later Verifying downstream PD is started... table new_ci_collation_test.t2 exists table new_ci_collation_test.t3 not exists for 1-th check, retry later go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ca788000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:15219, start at 2024-05-31 17:55:13.517627641 +0800 CST m=+1.955945886 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:13.525 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:13.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:13.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ca788000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:15219, start at 2024-05-31 17:55:13.517627641 +0800 CST m=+1.955945886 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:13.525 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:13.506 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:13.506 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384ca82c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:15295, start at 2024-05-31 17:55:13.589057501 +0800 CST m=+1.961349260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:13.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:13.597 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:13.597 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:13 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a498060c-be72-4bbf-97cd-9e65b28f558d {"id":"a498060c-be72-4bbf-97cd-9e65b28f558d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149311} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12f10788 a498060c-be72-4bbf-97cd-9e65b28f558d /tidb/cdc/default/default/upstream/7375100087745084139 {"id":7375100087745084139,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a498060c-be72-4bbf-97cd-9e65b28f558d {"id":"a498060c-be72-4bbf-97cd-9e65b28f558d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149311} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12f10788 a498060c-be72-4bbf-97cd-9e65b28f558d /tidb/cdc/default/default/upstream/7375100087745084139 {"id":7375100087745084139,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a498060c-be72-4bbf-97cd-9e65b28f558d {"id":"a498060c-be72-4bbf-97cd-9e65b28f558d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149311} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12f10788 a498060c-be72-4bbf-97cd-9e65b28f558d /tidb/cdc/default/default/upstream/7375100087745084139 {"id":7375100087745084139,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8780.out cli tso query --pd=http://127.0.0.1:2379 table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later [Fri May 31 17:55:14 CST 2024] <<<<<< START kafka consumer in changefeed_finish case >>>>>> table test.finish_mark not exists for 6-th check, retry later check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:15 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/586d41af-744d-49e1-82ca-bbc1451905ff {"id":"586d41af-744d-49e1-82ca-bbc1451905ff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149312} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12f42f8c 586d41af-744d-49e1-82ca-bbc1451905ff /tidb/cdc/default/default/upstream/7375100086301291915 {"id":7375100086301291915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/586d41af-744d-49e1-82ca-bbc1451905ff {"id":"586d41af-744d-49e1-82ca-bbc1451905ff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149312} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12f42f8c 586d41af-744d-49e1-82ca-bbc1451905ff /tidb/cdc/default/default/upstream/7375100086301291915 {"id":7375100086301291915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/586d41af-744d-49e1-82ca-bbc1451905ff {"id":"586d41af-744d-49e1-82ca-bbc1451905ff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149312} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce12f42f8c 586d41af-744d-49e1-82ca-bbc1451905ff /tidb/cdc/default/default/upstream/7375100086301291915 {"id":7375100086301291915,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.many_pk_or_uk.cli.2953.out cli changefeed create --start-ts=450140388868751361 '--sink-uri=kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-15669?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 57008fbc-625d-4dd5-85cf-ed657091745b Info: {"upstream_id":7375100086301291915,"namespace":"default","id":"57008fbc-625d-4dd5-85cf-ed657091745b","sink_uri":"kafka://127.0.0.1:9092/ticdc-many-pk-or-uk-test-15669?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:55:15.558417513+08:00","start_ts":450140388868751361,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140388868751361,"checkpoint_ts":450140388868751361,"checkpoint_time":"2024-05-31 17:55:10.565"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='450140389853626369 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140389853626369 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:55:15 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.88168818.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table new_ci_collation_test.t3 exists table new_ci_collation_test.t4 not exists for 1-th check, retry later check diff successfully [Fri May 31 17:55:15 CST 2024] <<<<<< START cdc server in kafka_column_selector case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.1676716769.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 7-th check, retry later wait process cdc.test exit for 1-th time... + set +x [Fri May 31 17:55:16 CST 2024] <<<<<< START kafka consumer in many_pk_or_uk case >>>>>> go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading go.uber.org/zap v1.27.0 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240520031014-397a460dd067 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240520031014-397a460dd067 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:55:17 CST 2024] <<<<<< run test case tiflash success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table new_ci_collation_test.t4 exists table new_ci_collation_test.t5 not exists for 1-th check, retry later table test.finish_mark not exists for 8-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:18 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7535849e-d083-4e83-8e8b-c630b17ac27b {"id":"7535849e-d083-4e83-8e8b-c630b17ac27b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149316} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13050393 7535849e-d083-4e83-8e8b-c630b17ac27b /tidb/cdc/default/default/upstream/7375100106302387520 {"id":7375100106302387520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7535849e-d083-4e83-8e8b-c630b17ac27b {"id":"7535849e-d083-4e83-8e8b-c630b17ac27b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149316} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13050393 7535849e-d083-4e83-8e8b-c630b17ac27b /tidb/cdc/default/default/upstream/7375100106302387520 {"id":7375100106302387520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7535849e-d083-4e83-8e8b-c630b17ac27b {"id":"7535849e-d083-4e83-8e8b-c630b17ac27b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149316} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13050393 7535849e-d083-4e83-8e8b-c630b17ac27b /tidb/cdc/default/default/upstream/7375100106302387520 {"id":7375100106302387520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.8861.out cli changefeed create --start-ts=450140389853626369 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-source-test-29848?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 8b7c184d-c9b5-41fe-8cff-3cc8207b7834 Info: {"upstream_id":7375100106302387520,"namespace":"default","id":"8b7c184d-c9b5-41fe-8cff-3cc8207b7834","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-source-test-29848?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:55:19.312491241+08:00","start_ts":450140389853626369,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140389853626369,"checkpoint_ts":450140389853626369,"checkpoint_time":"2024-05-31 17:55:14.322"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:19 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/242ceee9-5c3c-4bf6-9110-21ad8882b8b8 {"id":"242ceee9-5c3c-4bf6-9110-21ad8882b8b8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149316} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13123a81 242ceee9-5c3c-4bf6-9110-21ad8882b8b8 /tidb/cdc/default/default/upstream/7375100121194260705 {"id":7375100121194260705,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/242ceee9-5c3c-4bf6-9110-21ad8882b8b8 {"id":"242ceee9-5c3c-4bf6-9110-21ad8882b8b8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149316} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13123a81 242ceee9-5c3c-4bf6-9110-21ad8882b8b8 /tidb/cdc/default/default/upstream/7375100121194260705 {"id":7375100121194260705,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/242ceee9-5c3c-4bf6-9110-21ad8882b8b8 {"id":"242ceee9-5c3c-4bf6-9110-21ad8882b8b8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149316} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13123a81 242ceee9-5c3c-4bf6-9110-21ad8882b8b8 /tidb/cdc/default/default/upstream/7375100121194260705 {"id":7375100121194260705,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector.cli.16826.out cli changefeed create --start-ts=450140390281969665 '--sink-uri=kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json&partition-num=1&enable-tidb-extension=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/conf/changefeed.toml Create changefeed successfully! ID: test Info: {"upstream_id":7375100121194260705,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-test?protocol=canal-json\u0026partition-num=1\u0026enable-tidb-extension=true","create_time":"2024-05-31T17:55:19.457346088+08:00","start_ts":450140390281969665,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"column_selectors":[{"matcher":["test.t1"],"columns":["a","b"]},{"matcher":["test.*"],"columns":["*","!b"]},{"matcher":["test1.t1"],"columns":["column*","!column1"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140390281969665,"checkpoint_ts":450140390281969665,"checkpoint_time":"2024-05-31 17:55:15.956"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... go: downloading golang.org/x/text v0.15.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240516031922-38e0dca30c8c go: downloading github.com/spf13/pflag v1.0.5 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/google/uuid v1.6.0 go: downloading golang.org/x/net v0.25.0 go: downloading google.golang.org/grpc v1.63.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading golang.org/x/sys v0.20.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.20.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384d0638000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:4353, start at 2024-05-31 17:55:19.583153028 +0800 CST m=+1.856565276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:19.591 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:19.566 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:19.566 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384d0638000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:4353, start at 2024-05-31 17:55:19.583153028 +0800 CST m=+1.856565276 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:19.591 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:19.566 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:19.566 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384d07fc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:4414, start at 2024-05-31 17:55:19.727113899 +0800 CST m=+1.928603360 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:19.735 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:19.729 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:19.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT) go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 table test.finish_mark exists check diff successfully + set +x [Fri May 31 17:55:20 CST 2024] <<<<<< START kafka consumer in multi_source case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384d13540014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:7114, start at 2024-05-31 17:55:20.454612848 +0800 CST m=+1.877828430 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:20.464 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:20.455 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:20.455 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384d13540014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:7114, start at 2024-05-31 17:55:20.454612848 +0800 CST m=+1.877828430 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:20.464 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:20.455 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:20.455 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384d13f00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:7189, start at 2024-05-31 17:55:20.459812676 +0800 CST m=+1.827539561 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:20.466 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:20.444 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:20.444 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240520031014-397a460dd067 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240520031014-397a460dd067 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 + set +x wait process cdc.test exit for 1-th time... table new_ci_collation_test.t5 exists check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... check diff successfully wait process cdc.test exit for 3-th time... [Fri May 31 17:55:22 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.57245726.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri May 31 17:55:22 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> Verifying downstream PD is started... [Fri May 31 17:55:22 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.84968498.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 check diff failed 2-th time, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/json-iterator/go v1.1.12 go: downloading golang.org/x/text v0.15.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240516031922-38e0dca30c8c go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/stretchr/testify v1.9.0 go: downloading google.golang.org/grpc v1.63.2 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/otiai10/copy v1.2.0 go: downloading golang.org/x/net v0.25.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/google/btree v1.1.2 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.20.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading golang.org/x/tools v0.21.0 go: downloading golang.org/x/sys v0.20.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:25 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40e1ebb5-e0bf-4579-a4ba-488cc71c80e2 {"id":"40e1ebb5-e0bf-4579-a4ba-488cc71c80e2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149322} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13243d80 40e1ebb5-e0bf-4579-a4ba-488cc71c80e2 /tidb/cdc/default/default/upstream/7375100140076267628 {"id":7375100140076267628,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40e1ebb5-e0bf-4579-a4ba-488cc71c80e2 {"id":"40e1ebb5-e0bf-4579-a4ba-488cc71c80e2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149322} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13243d80 40e1ebb5-e0bf-4579-a4ba-488cc71c80e2 /tidb/cdc/default/default/upstream/7375100140076267628 {"id":7375100140076267628,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40e1ebb5-e0bf-4579-a4ba-488cc71c80e2 {"id":"40e1ebb5-e0bf-4579-a4ba-488cc71c80e2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149322} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13243d80 40e1ebb5-e0bf-4579-a4ba-488cc71c80e2 /tidb/cdc/default/default/upstream/7375100140076267628 {"id":7375100140076267628,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Starting build checksum checker... go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240520031014-397a460dd067 go: downloading golang.org/x/net v0.25.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240520031014-397a460dd067 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading google.golang.org/grpc v1.63.2 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240516031922-38e0dca30c8c go: downloading github.com/gogo/protobuf v1.3.2 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/stretchr/testify v1.9.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/containerd/cgroups v1.0.4 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 check diff successfully [Fri May 31 17:55:25 CST 2024] <<<<<< START kafka consumer in changefeed_reconstruct case >>>>>> go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/text v0.15.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading golang.org/x/sys v0.20.0 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 wait process cdc.test exit for 1-th time... go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading golang.org/x/oauth2 v0.20.0 go: downloading google.golang.org/api v0.170.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda check diff failed 3-th time, retry later go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "recordcount"="10" "operationcount"="0" "readallfields"="true" "mysql.db"="sink_retry" "scanproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "insertproportion"="0" "requestdistribution"="uniform" "updateproportion"="0" "mysql.port"="4000" "readproportion"="0" "mysql.user"="root" "threadcount"="2" "dotransactions"="false" ********************************************** Run finished, takes 5.299327ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2492.4, Avg(us): 890, Min(us): 618, Max(us): 2111, 95th(us): 3000, 99th(us): 3000 Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:25 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6d57b8d-5b65-4097-9a0b-7c896c843963 {"id":"b6d57b8d-5b65-4097-9a0b-7c896c843963","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce132b9e7e b6d57b8d-5b65-4097-9a0b-7c896c843963 /tidb/cdc/default/default/upstream/7375100147348505938 {"id":7375100147348505938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6d57b8d-5b65-4097-9a0b-7c896c843963 {"id":"b6d57b8d-5b65-4097-9a0b-7c896c843963","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce132b9e7e b6d57b8d-5b65-4097-9a0b-7c896c843963 /tidb/cdc/default/default/upstream/7375100147348505938 {"id":7375100147348505938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b6d57b8d-5b65-4097-9a0b-7c896c843963 {"id":"b6d57b8d-5b65-4097-9a0b-7c896c843963","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149323} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce132b9e7e b6d57b8d-5b65-4097-9a0b-7c896c843963 /tidb/cdc/default/default/upstream/7375100147348505938 {"id":7375100147348505938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.8561.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c=ddl-only-block-related-table go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading golang.org/x/term v0.20.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7375100147348505938,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:55:26.411100565+08:00","start_ts":450140392982052867,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140392982052867,"checkpoint_ts":450140392982052867,"checkpoint_time":"2024-05-31 17:55:26.256"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "mysql.db"="changefeed_reconstruct" "updateproportion"="0" "dotransactions"="false" "insertproportion"="0" "recordcount"="50" "mysql.user"="root" "workload"="core" "mysql.port"="4000" "scanproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "readallfields"="true" "threadcount"="4" "readproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 22.210776ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2913.4, Avg(us): 1761, Min(us): 1044, Max(us): 5722, 95th(us): 6000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:55:26 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> go: downloading github.com/ardielle/ardielle-go v1.5.2 check diff failed 4-th time, retry later + set +x [Fri May 31 17:55:27 CST 2024] <<<<<< START kafka consumer in ddl_only_block_related_table case >>>>>> table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff successfully check diff failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384d97e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:4612, start at 2024-05-31 17:55:28.926886041 +0800 CST m=+2.887357629 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:28.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:28.938 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:28.938 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384d97e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:4612, start at 2024-05-31 17:55:28.926886041 +0800 CST m=+2.887357629 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:28.935 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:28.938 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:28.938 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384d9d400005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:4697, start at 2024-05-31 17:55:29.238696434 +0800 CST m=+3.137998201 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:29.247 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:29.232 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:29.232 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later check diff successfully wait process 5729 exit for 1-th time... wait process 5729 exit for 2-th time... check diff successfully /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5729) - No such process wait process 5729 exit for 3-th time... process 5729 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Fri May 31 17:55:32 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.61476149.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later [Fri May 31 17:55:32 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.60116013.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:55:33 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:35 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4756241-ac5f-4ce5-8399-84060649b57a {"id":"d4756241-ac5f-4ce5-8399-84060649b57a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1342998b d4756241-ac5f-4ce5-8399-84060649b57a /tidb/cdc/default/default/upstream/7375100180038134970 {"id":7375100180038134970,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4756241-ac5f-4ce5-8399-84060649b57a {"id":"d4756241-ac5f-4ce5-8399-84060649b57a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1342998b d4756241-ac5f-4ce5-8399-84060649b57a /tidb/cdc/default/default/upstream/7375100180038134970 {"id":7375100180038134970,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d4756241-ac5f-4ce5-8399-84060649b57a {"id":"d4756241-ac5f-4ce5-8399-84060649b57a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1342998b d4756241-ac5f-4ce5-8399-84060649b57a /tidb/cdc/default/default/upstream/7375100180038134970 {"id":7375100180038134970,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:55:35 CST 2024] <<<<<< START kafka consumer in changefeed_pause_resume case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:35 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/e563aec7-0784-4413-8273-eca267b63c9a {UpstreamID:7375100140076267628 Namespace:default ID:e563aec7-0784-4413-8273-eca267b63c9a SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-10663?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:55:25.382125656 +0800 CST StartTs:450140392709423109 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003bc4120 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140392748744706} {CheckpointTs:450140393902178305 MinTableBarrierTs:450140395212898306 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450140393902178305, checkpointTs: 450140393902178305, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d53cef91-d5df-4e20-b467-4f545a244f3d {"id":"d53cef91-d5df-4e20-b467-4f545a244f3d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13243e28 d53cef91-d5df-4e20-b467-4f545a244f3d /tidb/cdc/default/default/changefeed/info/e563aec7-0784-4413-8273-eca267b63c9a {"upstream-id":7375100140076267628,"namespace":"default","changefeed-id":"e563aec7-0784-4413-8273-eca267b63c9a","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-10663?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:55:25.382125656+08:00","start-ts":450140392709423109,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140392748744706} /tidb/cdc/default/default/changefeed/status/e563aec7-0784-4413-8273-eca267b63c9a {"checkpoint-ts":450140393902178305,"min-table-barrier-ts":450140395212898306,"admin-job-type":0} /tidb/cdc/default/default/task/position/d53cef91-d5df-4e20-b467-4f545a244f3d/e563aec7-0784-4413-8273-eca267b63c9a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100140076267628 {"id":7375100140076267628,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e563aec7-0784-4413-8273-eca267b63c9a {UpstreamID:7375100140076267628 Namespace:default ID:e563aec7-0784-4413-8273-eca267b63c9a SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-10663?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:55:25.382125656 +0800 CST StartTs:450140392709423109 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003bc4120 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140392748744706} {CheckpointTs:450140393902178305 MinTableBarrierTs:450140395212898306 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450140393902178305, checkpointTs: 450140393902178305, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d53cef91-d5df-4e20-b467-4f545a244f3d {"id":"d53cef91-d5df-4e20-b467-4f545a244f3d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13243e28 d53cef91-d5df-4e20-b467-4f545a244f3d /tidb/cdc/default/default/changefeed/info/e563aec7-0784-4413-8273-eca267b63c9a {"upstream-id":7375100140076267628,"namespace":"default","changefeed-id":"e563aec7-0784-4413-8273-eca267b63c9a","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-10663?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:55:25.382125656+08:00","start-ts":450140392709423109,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140392748744706} /tidb/cdc/default/default/changefeed/status/e563aec7-0784-4413-8273-eca267b63c9a {"checkpoint-ts":450140393902178305,"min-table-barrier-ts":450140395212898306,"admin-job-type":0} /tidb/cdc/default/default/task/position/d53cef91-d5df-4e20-b467-4f545a244f3d/e563aec7-0784-4413-8273-eca267b63c9a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100140076267628 {"id":7375100140076267628,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/e563aec7-0784-4413-8273-eca267b63c9a {UpstreamID:7375100140076267628 Namespace:default ID:e563aec7-0784-4413-8273-eca267b63c9a SinkURI:kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-10663?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:55:25.382125656 +0800 CST StartTs:450140392709423109 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003bc4120 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140392748744706} {CheckpointTs:450140393902178305 MinTableBarrierTs:450140395212898306 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450140393902178305, checkpointTs: 450140393902178305, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d53cef91-d5df-4e20-b467-4f545a244f3d {"id":"d53cef91-d5df-4e20-b467-4f545a244f3d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149332} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13243e28 d53cef91-d5df-4e20-b467-4f545a244f3d /tidb/cdc/default/default/changefeed/info/e563aec7-0784-4413-8273-eca267b63c9a {"upstream-id":7375100140076267628,"namespace":"default","changefeed-id":"e563aec7-0784-4413-8273-eca267b63c9a","sink-uri":"kafka://127.0.0.1:9092/ticdc-changefeed-reconstruct-10663?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:55:25.382125656+08:00","start-ts":450140392709423109,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140392748744706} /tidb/cdc/default/default/changefeed/status/e563aec7-0784-4413-8273-eca267b63c9a {"checkpoint-ts":450140393902178305,"min-table-barrier-ts":450140395212898306,"admin-job-type":0} /tidb/cdc/default/default/task/position/d53cef91-d5df-4e20-b467-4f545a244f3d/e563aec7-0784-4413-8273-eca267b63c9a {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100140076267628 {"id":7375100140076267628,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id table changefeed_pause_resume.t1 not exists for 1-th check, retry later table ddl_only_block_related_table.finish_mark exists wait process 8501 exit for 1-th time... wait process 8501 exit for 2-th time... "id": "d53cef91-d5df-4e20-b467-4f545a244f3d", "cluster-id": "default" run task successfully /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8501) - No such process wait process 8501 exit for 3-th time... process 8501 already exit [Fri May 31 17:55:37 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.86848686.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_pause_resume.t1 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_handle_key_only Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... capture_id: d53cef91-d5df-4e20-b467-4f545a244f3d check_processor_table_count http://127.0.0.1:2379 e563aec7-0784-4413-8273-eca267b63c9a d53cef91-d5df-4e20-b467-4f545a244f3d 1 run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: kafka... <<================= check_processor_table_count http://127.0.0.1:2379 e563aec7-0784-4413-8273-eca267b63c9a d53cef91-d5df-4e20-b467-4f545a244f3d 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:40 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7375100147348505938 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:55:26.411100565 +0800 CST StartTs:450140392982052867 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002cf47e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140393021112321} {CheckpointTs:450140396481413126 MinTableBarrierTs:450140396481413126 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0b4c9c5-4873-41c4-8214-efdbfe7ab264 {"id":"f0b4c9c5-4873-41c4-8214-efdbfe7ab264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce132b9f4f f0b4c9c5-4873-41c4-8214-efdbfe7ab264 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375100147348505938,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:55:26.411100565+08:00","start-ts":450140392982052867,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140393021112321} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450140396481413126,"min-table-barrier-ts":450140396743819272,"admin-job-type":0} /tidb/cdc/default/default/task/position/f0b4c9c5-4873-41c4-8214-efdbfe7ab264/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100147348505938 {"id":7375100147348505938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7375100147348505938 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:55:26.411100565 +0800 CST StartTs:450140392982052867 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002cf47e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140393021112321} {CheckpointTs:450140396481413126 MinTableBarrierTs:450140396481413126 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0b4c9c5-4873-41c4-8214-efdbfe7ab264 {"id":"f0b4c9c5-4873-41c4-8214-efdbfe7ab264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce132b9f4f f0b4c9c5-4873-41c4-8214-efdbfe7ab264 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375100147348505938,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:55:26.411100565+08:00","start-ts":450140392982052867,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140393021112321} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450140396481413126,"min-table-barrier-ts":450140396743819272,"admin-job-type":0} /tidb/cdc/default/default/task/position/f0b4c9c5-4873-41c4-8214-efdbfe7ab264/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100147348505938 {"id":7375100147348505938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7375100147348505938 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:55:26.411100565 +0800 CST StartTs:450140392982052867 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002cf47e0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140393021112321} {CheckpointTs:450140396481413126 MinTableBarrierTs:450140396481413126 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450140396743819272, checkpointTs: 450140396481413126, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f0b4c9c5-4873-41c4-8214-efdbfe7ab264 {"id":"f0b4c9c5-4873-41c4-8214-efdbfe7ab264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149337} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce132b9f4f f0b4c9c5-4873-41c4-8214-efdbfe7ab264 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375100147348505938,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:55:26.411100565+08:00","start-ts":450140392982052867,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"syn+ grep -q 'etcd info' ced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140393021112321} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450140396481413126,"min-table-barrier-ts":450140396743819272,"admin-job-type":0} /tidb/cdc/default/default/task/position/f0b4c9c5-4873-41c4-8214-efdbfe7ab264/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100147348505938 {"id":7375100147348505938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_ts_not_forward ddl-only-block-related-table check_processor_table_count http://127.0.0.1:2379 e563aec7-0784-4413-8273-eca267b63c9a d53cef91-d5df-4e20-b467-4f545a244f3d 0 Verifying downstream PD is started... run task successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table changefeed_pause_resume.t3 exists Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ***************** properties ***************** "threadcount"="4" "readproportion"="0" "mysql.db"="changefeed_reconstruct" "requestdistribution"="uniform" "recordcount"="50" "mysql.port"="4000" "scanproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "mysql.user"="root" "updateproportion"="0" "dotransactions"="false" "readallfields"="true" "insertproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 18.864962ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3254.0, Avg(us): 1504, Min(us): 1057, Max(us): 4141, 95th(us): 4000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later run task failed 1-th time, retry later check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_ts_not_forward ddl-only-block-related-table ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384e9ea8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:11313, start at 2024-05-31 17:55:45.717774508 +0800 CST m=+2.253748494 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:45.725 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:45.706 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:45.706 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384e9ea8000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:11313, start at 2024-05-31 17:55:45.717774508 +0800 CST m=+2.253748494 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:45.725 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:45.706 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:45.706 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384e9dcc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:11386, start at 2024-05-31 17:55:45.701790852 +0800 CST m=+2.172032643 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:45.710 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:45.702 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:45.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:55:48 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384eb0a00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b, pid:4181, start at 2024-05-31 17:55:46.878682752 +0800 CST m=+2.031479116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:46.885 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:46.856 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:46.856 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384eb0a00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b, pid:4181, start at 2024-05-31 17:55:46.878682752 +0800 CST m=+2.031479116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:46.885 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:46.856 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:46.856 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384eaf0c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b, pid:4255, start at 2024-05-31 17:55:46.765992087 +0800 CST m=+1.861836475 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:46.775 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:46.755 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:46.755 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/cdc Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... check diff failed 4-th time, retry later [Fri May 31 17:55:49 CST 2024] <<<<<< START cdc server in kafka_simple_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.1276812770.out server --log-file /tmp/tidb_cdc_test/kafka_simple_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_handle_key_only/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Fri May 31 17:55:51 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.56365638.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:52 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/32d54e6e-fc7a-41a6-8eb0-96b50f64e6a2 {"id":"32d54e6e-fc7a-41a6-8eb0-96b50f64e6a2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149350} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1388fa8a 32d54e6e-fc7a-41a6-8eb0-96b50f64e6a2 /tidb/cdc/default/default/upstream/7375100261250150202 {"id":7375100261250150202,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/32d54e6e-fc7a-41a6-8eb0-96b50f64e6a2 {"id":"32d54e6e-fc7a-41a6-8eb0-96b50f64e6a2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149350} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1388fa8a 32d54e6e-fc7a-41a6-8eb0-96b50f64e6a2 /tidb/cdc/default/default/upstream/7375100261250150202 {"id":7375100261250150202,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/32d54e6e-fc7a-41a6-8eb0-96b50f64e6a2 {"id":"32d54e6e-fc7a-41a6-8eb0-96b50f64e6a2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149350} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1388fa8a 32d54e6e-fc7a-41a6-8eb0-96b50f64e6a2 /tidb/cdc/default/default/upstream/7375100261250150202 {"id":7375100261250150202,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12820.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... check diff successfully Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:55:54 GMT < Content-Length: 859 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84ab6b42-1bdf-4ea8-b29d-0479fabcaa8b {"id":"84ab6b42-1bdf-4ea8-b29d-0479fabcaa8b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149351} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce138c4886 84ab6b42-1bdf-4ea8-b29d-0479fabcaa8b /tidb/cdc/default/default/upstream/7375100261406940518 {"id":7375100261406940518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84ab6b42-1bdf-4ea8-b29d-0479fabcaa8b {"id":"84ab6b42-1bdf-4ea8-b29d-0479fabcaa8b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149351} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce138c4886 84ab6b42-1bdf-4ea8-b29d-0479fabcaa8b /tidb/cdc/default/default/upstream/7375100261406940518 {"id":7375100261406940518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/84ab6b42-1bdf-4ea8-b29d-0479fabcaa8b {"id":"84ab6b42-1bdf-4ea8-b29d-0479fabcaa8b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149351} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce138c4886 84ab6b42-1bdf-4ea8-b29d-0479fabcaa8b /tidb/cdc/default/default/upstream/7375100261406940518 {"id":7375100261406940518,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 9dd2824e-dd1f-4e08-bbfa-9f1649f9f8d2 Info: {"upstream_id":7375100261406940518,"namespace":"default","id":"9dd2824e-dd1f-4e08-bbfa-9f1649f9f8d2","sink_uri":"kafka://127.0.0.1:9092/ticdc-region-merge-test-9417?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:55:54.473212454+08:00","start_ts":450140400322084871,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140400322084871,"checkpoint_ts":450140400322084871,"checkpoint_time":"2024-05-31 17:55:54.256"} [Fri May 31 17:55:54 CST 2024] <<<<<< START kafka consumer in region_merge case >>>>>> check diff failed 1-th time, retry later + set +x + tso='450140400059940869 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140400059940869 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12845.out cli changefeed create --start-ts=450140400059940869 '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-11204?protocol=simple' -c simple-handle-key-only --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: simple-handle-key-only Info: {"upstream_id":7375100261250150202,"namespace":"default","id":"simple-handle-key-only","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-11204?protocol=simple","create_time":"2024-05-31T17:55:55.27845801+08:00","start_ts":450140400059940869,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140400059940869,"checkpoint_ts":450140400059940869,"checkpoint_time":"2024-05-31 17:55:53.256"} PASS VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384f35140013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2, pid:4378, start at 2024-05-31 17:55:55.36702797 +0800 CST m=+1.890357977 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:55.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:55.333 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:55.333 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384f35140013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2, pid:4378, start at 2024-05-31 17:55:55.36702797 +0800 CST m=+1.890357977 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:55.374 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:55.333 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:55.333 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384f35f00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2, pid:4454, start at 2024-05-31 17:55:55.438008807 +0800 CST m=+1.905891223 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:57:55.445 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:55:55.438 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:45:55.438 +0800 All versions after safe point can be accessed. (DO NOT EDIT) coverage: 2.6% of statements in github.com/pingcap/tiflow/... split_and_random_merge scale: 20 Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... check diff failed 2-th time, retry later + set +x table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:55:57 CST 2024] <<<<<< run test case sink_retry success! >>>>>> [Fri May 31 17:55:57 CST 2024] <<<<<< START cdc server in cdc case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.58995901.out server --log-file /tmp/tidb_cdc_test/cdc/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:00 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf044fa7-b82f-4fe8-83ea-f0cb915f84a4 {"id":"cf044fa7-b82f-4fe8-83ea-f0cb915f84a4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149358} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13b40f80 cf044fa7-b82f-4fe8-83ea-f0cb915f84a4 /tidb/cdc/default/default/upstream/7375100301381544032 {"id":7375100301381544032,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf044fa7-b82f-4fe8-83ea-f0cb915f84a4 {"id":"cf044fa7-b82f-4fe8-83ea-f0cb915f84a4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149358} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13b40f80 cf044fa7-b82f-4fe8-83ea-f0cb915f84a4 /tidb/cdc/default/default/upstream/7375100301381544032 {"id":7375100301381544032,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cf044fa7-b82f-4fe8-83ea-f0cb915f84a4 {"id":"cf044fa7-b82f-4fe8-83ea-f0cb915f84a4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149358} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13b40f80 cf044fa7-b82f-4fe8-83ea-f0cb915f84a4 /tidb/cdc/default/default/upstream/7375100301381544032 {"id":7375100301381544032,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc.cli.5964.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cdc-test-19644?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config /tmp/tidb_cdc_test/cdc/pulsar_test.toml check diff failed 4-th time, retry later Create changefeed successfully! ID: 004dd880-2701-4843-a353-23e7a6f743f9 Info: {"upstream_id":7375100301381544032,"namespace":"default","id":"004dd880-2701-4843-a353-23e7a6f743f9","sink_uri":"kafka://127.0.0.1:9092/ticdc-cdc-test-19644?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:56:01.398473871+08:00","start_ts":450140402151325701,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140402151325701,"checkpoint_ts":450140402151325701,"checkpoint_time":"2024-05-31 17:56:01.234"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... *************************** 1. row *************************** count(distinct region_id): 1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12914.out cli changefeed pause -c simple-handle-key-only PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384f9d700010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:4257, start at 2024-05-31 17:56:02.029899086 +0800 CST m=+1.917284909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:02.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:02.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:02.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384f9d700010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:4257, start at 2024-05-31 17:56:02.029899086 +0800 CST m=+1.917284909 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:02.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:02.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:02.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f384f9e300004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:4344, start at 2024-05-31 17:56:02.061087658 +0800 CST m=+1.892518419 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:02.069 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:02.060 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:02.060 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x [Fri May 31 17:56:02 CST 2024] <<<<<< START kafka consumer in cdc case >>>>>> go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240520031014-397a460dd067 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240520031014-397a460dd067 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading go.uber.org/multierr v1.11.0 check diff failed 5-th time, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12937.out cli changefeed update -c simple-handle-key-only '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-11204?protocol=simple&max-message-bytes=700' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/conf/changefeed.toml --no-confirm start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/simple-handle-key-only-11204?protocol=simple To:kafka://127.0.0.1:9092/simple-handle-key-only-11204?protocol=simple&max-message-bytes=700} {Type:update Path:[Config SyncPointInterval] From: To:0xc0035a1848} {Type:update Path:[Config SyncPointRetention] From: To:0xc0035a1858} {Type:update Path:[Config Consistent] From: To:0xc001155dc0} Update changefeed config successfully! ID: simple-handle-key-only Info: {"upstream_id":7375100261250150202,"namespace":"default","id":"simple-handle-key-only","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-11204?protocol=simple\u0026max-message-bytes=700","create_time":"2024-05-31T17:55:55.27845801+08:00","start_ts":450140400059940869,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":0,"checkpoint_ts":450140402222628872,"checkpoint_time":"2024-05-31 17:56:01.506"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff successfully [Fri May 31 17:56:04 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.57865788.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading golang.org/x/text v0.15.0 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240516031922-38e0dca30c8c go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/tikv/pd/client v0.0.0-20240430080403-1679dbca25b3 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/spf13/pflag v1.0.5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/docker/go-units v0.5.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading golang.org/x/tools v0.21.0 go: downloading github.com/shirou/gopsutil/v3 v3.24.4 go: downloading github.com/stretchr/testify v1.9.0 go: downloading google.golang.org/grpc v1.63.2 go: downloading golang.org/x/net v0.25.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/google/btree v1.1.2 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading golang.org/x/sys v0.20.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/oauth2 v0.20.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/golang/snappy v0.0.4 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading golang.org/x/crypto v0.23.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/common v0.53.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading cloud.google.com/go/compute/metadata v0.3.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only.cli.12960.out cli changefeed resume -c simple-handle-key-only go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 split_and_random_merge scale: 40 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x table test.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:07 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/acd59307-8310-489b-a929-5334418f8e5e {"id":"acd59307-8310-489b-a929-5334418f8e5e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149365} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13cdf2c8 acd59307-8310-489b-a929-5334418f8e5e /tidb/cdc/default/default/upstream/7375100333941239984 {"id":7375100333941239984,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/acd59307-8310-489b-a929-5334418f8e5e {"id":"acd59307-8310-489b-a929-5334418f8e5e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149365} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13cdf2c8 acd59307-8310-489b-a929-5334418f8e5e /tidb/cdc/default/default/upstream/7375100333941239984 {"id":7375100333941239984,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/acd59307-8310-489b-a929-5334418f8e5e {"id":"acd59307-8310-489b-a929-5334418f8e5e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149365} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13cdf2c8 acd59307-8310-489b-a929-5334418f8e5e /tidb/cdc/default/default/upstream/7375100333941239984 {"id":7375100333941239984,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.5846.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-batch-add-table-test-8438?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 2382a770-dd12-4a52-bfdf-ad1fa4f8ec75 Info: {"upstream_id":7375100333941239984,"namespace":"default","id":"2382a770-dd12-4a52-bfdf-ad1fa4f8ec75","sink_uri":"kafka://127.0.0.1:9092/ticdc-batch-add-table-test-8438?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:56:08.415646473+08:00","start_ts":450140403993411587,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140403993411587,"checkpoint_ts":450140403993411587,"checkpoint_time":"2024-05-31 17:56:08.261"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 table test.finish_mark not exists for 2-th check, retry later check diff failed 3-th time, retry later go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd + set +x [Fri May 31 17:56:11 CST 2024] <<<<<< START kafka consumer in batch_add_table case >>>>>> table batch_add_table.finish_mark not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385027ec0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:7450, start at 2024-05-31 17:56:10.919208197 +0800 CST m=+1.899819895 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:10.927 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:10.926 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:10.926 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385027ec0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:7450, start at 2024-05-31 17:56:10.919208197 +0800 CST m=+1.899819895 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:10.927 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:10.926 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:10.926 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385027e40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:7526, start at 2024-05-31 17:56:10.894498151 +0800 CST m=+1.809239254 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:10.902 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:10.873 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:10.873 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... check diff failed 4-th time, retry later *************************** 1. row *************************** count(distinct region_id): 1 wait process cdc.test exit for 2-th time... table batch_add_table.finish_mark not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:56:13 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8826.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 5-th time, retry later + set +x + tso='450140405399814146 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140405399814146 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "workload"="core" "recordcount"="10" "scanproportion"="0" "readallfields"="true" "mysql.port"="4000" "mysql.user"="root" "insertproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" "updateproportion"="0" "readproportion"="0" "threadcount"="2" "dotransactions"="false" "requestdistribution"="uniform" "mysql.db"="multi_capture_1" ********************************************** Run finished, takes 9.140276ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1967.9, Avg(us): 1743, Min(us): 1051, Max(us): 3958, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.port"="4000" "dotransactions"="false" "updateproportion"="0" "threadcount"="2" "mysql.db"="multi_capture_2" "readallfields"="true" "mysql.user"="root" "insertproportion"="0" "readproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "recordcount"="10" "scanproportion"="0" "workload"="core" ********************************************** Run finished, takes 9.924795ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1744.7, Avg(us): 1863, Min(us): 1134, Max(us): 4160, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "scanproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "updateproportion"="0" "dotransactions"="false" "readallfields"="true" "mysql.user"="root" "readproportion"="0" "workload"="core" "mysql.db"="multi_capture_3" "mysql.port"="4000" "recordcount"="10" "operationcount"="0" "insertproportion"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.805595ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1788.0, Avg(us): 1875, Min(us): 1175, Max(us): 4116, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "readallfields"="true" "threadcount"="2" "recordcount"="10" "workload"="core" "mysql.user"="root" "insertproportion"="0" "operationcount"="0" "readproportion"="0" "scanproportion"="0" "mysql.port"="4000" "dotransactions"="false" "updateproportion"="0" "mysql.db"="multi_capture_4" ********************************************** Run finished, takes 9.925734ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1801.0, Avg(us): 1895, Min(us): 1175, Max(us): 4217, 95th(us): 5000, 99th(us): 5000 [Fri May 31 17:56:15 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.89388940.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... check diff failed 1-th time, retry later split_and_random_merge scale: 80 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:18 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ef33276-feb1-4d5f-88a7-c0fa24017f17 {"id":"8ef33276-feb1-4d5f-88a7-c0fa24017f17","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149375} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdee5 8ef33276-feb1-4d5f-88a7-c0fa24017f17 /tidb/cdc/default/default/upstream/7375100366531331668 {"id":7375100366531331668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ef33276-feb1-4d5f-88a7-c0fa24017f17 {"id":"8ef33276-feb1-4d5f-88a7-c0fa24017f17","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149375} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdee5 8ef33276-feb1-4d5f-88a7-c0fa24017f17 /tidb/cdc/default/default/upstream/7375100366531331668 {"id":7375100366531331668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8ef33276-feb1-4d5f-88a7-c0fa24017f17 {"id":"8ef33276-feb1-4d5f-88a7-c0fa24017f17","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149375} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdee5 8ef33276-feb1-4d5f-88a7-c0fa24017f17 /tidb/cdc/default/default/upstream/7375100366531331668 {"id":7375100366531331668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:56:18 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.89918993.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:56:15.649 +08:00] [INFO] [main.go:99] ["running ddl test: 1 modifyColumnDefaultValueDDL2"] [2024/05/31 17:56:15.649 +08:00] [INFO] [main.go:99] ["running ddl test: 0 modifyColumnDefaultValueDDL1"] [2024/05/31 17:56:15.702 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb8883e5a_8a00_4c7e_820e_b8e25255de5a"] [2024/05/31 17:56:15.709 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs726ed13e_dd48_4ec6_86d6_43c4eaf6d355"] [2024/05/31 17:56:15.734 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3173f7bf_1771_47ac_bd4e_e4b680c15603"] [2024/05/31 17:56:15.771 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs590b3361_86c2_4011_a07d_1044d0e27aa9"] [2024/05/31 17:56:15.779 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs4aecaa2b_5c57_46e5_b565_022a0db6edef"] [2024/05/31 17:56:15.805 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLse270f282_44c5_443a_94a5_10a03beca70d"] [2024/05/31 17:56:15.843 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6fdb3fc0_3d11_4651_b139_9e699ce520ac"] [2024/05/31 17:56:15.852 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf655ee4e_8510_47f7_aa3f_9f4377630c26"] [2024/05/31 17:56:16.186 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:16.258 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:16.260 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:16.263 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:16.285 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:16.291 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:16.381 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:16.383 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:16.384 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:16.389 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:16.395 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/31 17:56:16.405 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:16.409 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:16.472 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:16.472 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:16.473 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:16.475 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:16.497 +08:00] [INFO] [main.go:178] ["1 insert success: 100"] [2024/05/31 17:56:16.596 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/31 17:56:16.766 +08:00] [INFO] [main.go:178] ["0 insert success: 100"] [2024/05/31 17:56:16.779 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:16.858 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:16.860 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:16.861 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:16.884 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:16.884 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:16.975 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:16.977 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:16.985 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:16.985 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:16.997 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/31 17:56:17.006 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:17.060 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:17.075 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:17.076 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:17.079 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:17.088 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:17.093 +08:00] [INFO] [main.go:178] ["1 insert success: 200"] [2024/05/31 17:56:17.378 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:17.458 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:17.473 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:17.503 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:17.554 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/31 17:56:17.559 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/31 17:56:17.562 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:17.562 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:17.578 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:17.588 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:17.595 +08:00] [INFO] [main.go:199] ["0 delete success: 100"] [2024/05/31 17:56:17.596 +08:00] [INFO] [main.go:178] ["0 insert success: 200"] [2024/05/31 17:56:17.599 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/31 17:56:17.658 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:17.668 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:17.671 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:17.677 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:17.692 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:17.694 +08:00] [INFO] [main.go:178] ["1 insert success: 300"] [2024/05/31 17:56:17.697 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:17.773 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:17.780 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:17.990 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:18.063 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:18.081 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:18.190 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:18.194 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:18.194 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:18.203 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:18.256 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:18.271 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/31 17:56:18.288 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:18.292 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:18.295 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:18.303 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:18.354 +08:00] [INFO] [main.go:178] ["1 insert success: 400"] [2024/05/31 17:56:18.359 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:18.374 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:18.407 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:18.409 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:18.412 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/31 17:56:18.485 +08:00] [INFO] [main.go:178] ["0 insert success: 300"] [2024/05/31 17:56:18.676 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:18.682 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:18.757 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:18.858 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:18.864 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:18.895 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:18.956 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:18.965 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:18.985 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:18.988 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/31 17:56:18.990 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:18.993 +08:00] [INFO] [main.go:178] ["1 insert success: 500"] [2024/05/31 17:56:19.002 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:19.057 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:19.068 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:19.081 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:19.102 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:19.108 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:19.375 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/31 17:56:19.379 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:19.379 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:19.380 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/31 17:56:19.455 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:19.455 +08:00] [INFO] [main.go:199] ["0 delete success: 200"] [2024/05/31 17:56:19.456 +08:00] [INFO] [main.go:178] ["0 insert success: 400"] [2024/05/31 17:56:19.489 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:19.497 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:19.578 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:19.591 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:19.600 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:19.662 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:19.665 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:19.670 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/31 17:56:19.687 +08:00] [INFO] [main.go:178] ["1 insert success: 600"] [2024/05/31 17:56:19.697 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:19.697 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:19.702 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:19.766 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:19.770 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:19.779 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:20.062 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:20.073 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:20.092 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:20.160 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:20.168 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:20.254 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:20.265 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/31 17:56:20.272 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:20.279 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:20.283 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:20.286 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:20.294 +08:00] [INFO] [main.go:178] ["0 insert success: 500"] [2024/05/31 17:56:20.294 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/31 17:56:20.362 +08:00] [INFO] [main.go:178] ["1 insert success: 700"] [2024/05/31 17:56:20.371 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:20.371 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:20.375 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:20.388 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:20.396 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:20.457 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:20.679 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:20.698 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:20.762 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:20.764 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:20.784 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:20.880 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:20.953 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:20.953 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:20.963 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:20.967 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:20.975 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/31 17:56:20.989 +08:00] [INFO] [main.go:178] ["1 insert success: 800"] [2024/05/31 17:56:21.056 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:21.059 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:21.064 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:21.076 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:21.087 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:21.100 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:21.172 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/31 17:56:21.177 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/31 17:56:21.192 +08:00] [INFO] [main.go:199] ["0 delete success: 300"] [2024/05/31 17:56:21.193 +08:00] [INFO] [main.go:178] ["0 insert success: 600"] [2024/05/31 17:56:21.383 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:56:21.470 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:56:21.495 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:56:21.554 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:56:21.571 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/31 17:56:21.589 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:56:21.597 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:56:21.599 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:56:21.600 +08:00] [INFO] [main.go:178] ["1 insert success: 900"] [2024/05/31 17:56:21.613 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:56:21.673 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:56:21.676 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:56:21.694 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:56:21.703 +08:00] [INFO] [main.go:178] ["72 insert success: 900"] [2024/05/31 17:56:21.864 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/31 17:56:21.917 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] [2024/05/31 17:56:21.922 +08:00] [INFO] [main.go:178] ["0 insert success: 700"] [2024/05/31 17:56:21.962 +08:00] [INFO] [main.go:178] ["1 insert success: 1000"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:21 GMT < Content-Length: 1271 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 {"id":"5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149379} /tidb/cdc/default/__cdc_meta__/capture/8ef33276-feb1-4d5f-88a7-c0fa24017f17 {"id":"8ef33276-feb1-4d5f-88a7-c0fa24017f17","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149375} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdee5 8ef33276-feb1-4d5f-88a7-c0fa24017f17 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdf20 5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 /tidb/cdc/default/default/upstream/7375100366531331668 {"id":7375100366531331668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 {"id":"5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149379} /tidb/cdc/default/__cdc_meta__/capture/8ef33276-feb1-4d5f-88a7-c0fa24017f17 {"id":"8ef33276-feb1-4d5f-88a7-c0fa24017f17","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149375} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdee5 8ef33276-feb1-4d5f-88a7-c0fa24017f17 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdf20 5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 /tidb/cdc/default/default/upstream/7375100366531331668 {"id":7375100366531331668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 {"id":"5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149379} /tidb/cdc/default/__cdc_meta__/capture/8ef33276-feb1-4d5f-88a7-c0fa24017f17 {"id":"8ef33276-feb1-4d5f-88a7-c0fa24017f17","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149375} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdee5 8ef33276-feb1-4d5f-88a7-c0fa24017f17 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdf20 5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 /tidb/cdc/default/default/upstream/7375100366531331668 {"id":7375100366531331668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:56:21 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.90389040.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:56:21.996 +08:00] [INFO] [main.go:178] ["73 insert success: 1000"] [2024/05/31 17:56:22.053 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs24b23eb3_32b2_4fd8_95e9_68243d037351"] [2024/05/31 17:56:22.092 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsaec72b6f_c861_4fdb_956f_e7341bc9eb8d"] [2024/05/31 17:56:22.099 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs58722bfe_e359_41d8_829c_fbf03b12eb2b"] [2024/05/31 17:56:22.139 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb37e8c48_e846_43d1_9a03_13eb1ed97222"] [2024/05/31 17:56:22.147 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs63acb7b6_882c_4a99_a320_adb99e8c00ce"] [2024/05/31 17:56:22.175 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsbfbd724f_6391_4ab1_8f6d_275727cd0338"] [2024/05/31 17:56:22.201 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs18a69c17_30ce_4373_bd88_61f0822dc891"] [2024/05/31 17:56:22.266 +08:00] [INFO] [main.go:178] ["1 insert success: 1100"] [2024/05/31 17:56:22.381 +08:00] [INFO] [main.go:199] ["0 delete success: 400"] [2024/05/31 17:56:22.382 +08:00] [INFO] [main.go:178] ["0 insert success: 800"] [2024/05/31 17:56:22.470 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:22.472 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:22.555 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:22.562 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:22.563 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:22.578 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:22.587 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs1ecbc071_253b_4699_b6a2_730a1598a051"] [2024/05/31 17:56:22.608 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:22.617 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:22.659 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:22.665 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:22.680 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:22.694 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:22.697 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:22.717 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:22.759 +08:00] [INFO] [main.go:178] ["1 insert success: 1200"] [2024/05/31 17:56:22.962 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:22.975 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:23.071 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:23.084 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:23.090 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:23.104 +08:00] [INFO] [main.go:178] ["0 insert success: 900"] [2024/05/31 17:56:23.105 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:23.159 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:23.169 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:23.183 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:23.196 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:23.257 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:23.272 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:23.279 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:23.290 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:23.297 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:23.303 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:23.354 +08:00] [INFO] [main.go:178] ["1 insert success: 1300"] [2024/05/31 17:56:23.558 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:56:23.569 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:23.654 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:23.657 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:23.679 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:23.692 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:23.702 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:23.704 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:23.712 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:23.768 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:23.787 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:23.793 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:23.799 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:23.803 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:23.858 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:23.860 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:23.995 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] check diff failed 4-th time, retry later [2024/05/31 17:56:24.077 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:24.106 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:24.154 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:24.195 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:24.199 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:24.204 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:24.209 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:24.255 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:24.285 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:24.304 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:24.313 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:24.315 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:24.322 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] *************************** 1. row *************************** count(distinct region_id): 1 [2024/05/31 17:56:24.370 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:24.372 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:24.480 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:24.503 +08:00] [INFO] [main.go:88] ["testGetDefaultValue take 8.853646835s"] [2024/05/31 17:56:24.557 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:24.598 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:24.599 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:24.681 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:24.684 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:24.698 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:24.710 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:24.712 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:24.788 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:24.812 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:24.853 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:24.861 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:24.871 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:24.885 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:24.889 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:24.971 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:25.009 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:25.087 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:25.088 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:25.175 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:25.186 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:25.201 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:25.204 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:24 GMT < Content-Length: 1750 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 {"id":"5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149379} /tidb/cdc/default/__cdc_meta__/capture/8ef33276-feb1-4d5f-88a7-c0fa24017f17 {"id":"8ef33276-feb1-4d5f-88a7-c0fa24017f17","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149375} /tidb/cdc/default/__cdc_meta__/capture/ea3259db-2485-45f1-b9c2-8a891828b8ca {"id":"ea3259db-2485-45f1-b9c2-8a891828b8ca","address":"127.0.0.1:8303","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149382} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdee5 8ef33276-feb1-4d5f-88a7-c0fa24017f17 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdf20 5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdf3d ea3259db-2485-45f1-b9c2-8a891828b8ca /tidb/cdc/default/default/upstream/7375100366531331668 {"id":7375100366531331668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 {"id":"5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149379} /tidb/cdc/default/__cdc_meta__/capture/8ef33276-feb1-4d5f-88a7-c0fa24017f17 {"id":"8ef33276-feb1-4d5f-88a7-c0fa24017f17","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149375} /tidb/cdc/default/__cdc_meta__/capture/ea3259db-2485-45f1-b9c2-8a891828b8ca {"id":"ea3259db-2485-45f1-b9c2-8a891828b8ca","address":"127.0.0.1:8303","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149382} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdee5 8ef33276-feb1-4d5f-88a7-c0fa24017f17 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdf20 5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdf3d ea3259db-2485-45f1-b9c2-8a891828b8ca /tidb/cdc/default/default/upstream/7375100366531331668 {"id":7375100366531331668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 {"id":"5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149379} /tidb/cdc/default/__cdc_meta__/capture/8ef33276-feb1-4d5f-88a7-c0fa24017f17 {"id":"8ef33276-feb1-4d5f-88a7-c0fa24017f17","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149375} /tidb/cdc/default/__cdc_meta__/capture/ea3259db-2485-45f1-b9c2-8a891828b8ca {"id":"ea3259db-2485-45f1-b9c2-8a891828b8ca","address":"127.0.0.1:8303","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149382} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdee5 8ef33276-feb1-4d5f-88a7-c0fa24017f17 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdf20 5c4b0fc6-6acd-404d-9e6a-3f64e6f5e151 /tidb/cdc/default/__cdc_meta__/owner/22318fce13ecdf3d ea3259db-2485-45f1-b9c2-8a891828b8ca /tidb/cdc/default/default/upstream/7375100366531331668 {"id":7375100366531331668,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.9083.out cli changefeed create --start-ts=450140405399814146 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-capture-test-18640?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8301 Create changefeed successfully! ID: 0d2673e5-8420-482b-8bb4-0e6088a9d40f Info: {"upstream_id":7375100366531331668,"namespace":"default","id":"0d2673e5-8420-482b-8bb4-0e6088a9d40f","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-capture-test-18640?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:56:25.219474435+08:00","start_ts":450140405399814146,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140405399814146,"checkpoint_ts":450140405399814146,"checkpoint_time":"2024-05-31 17:56:13.626"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/31 17:56:25.265 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:25.304 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:25.371 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:25.374 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:25.382 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:25.389 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:25.403 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:25.408 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:25.487 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:25.513 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh: line 1: 12998 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3850ef2c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:4287, start at 2024-05-31 17:56:23.657537337 +0800 CST m=+2.127768005 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:23.664 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:23.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:23.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3850ef2c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:4287, start at 2024-05-31 17:56:23.657537337 +0800 CST m=+2.127768005 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:23.664 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:23.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:23.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3850f16c0007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:4366, start at 2024-05-31 17:56:23.779901157 +0800 CST m=+2.191195295 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:23.788 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:23.771 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:23.771 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [2024/05/31 17:56:25.594 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:25.596 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:25.671 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:25.690 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:25.694 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:25.704 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:25.761 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:25.795 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/31 17:56:25.860 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:25.870 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:25.888 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:25.899 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:25.911 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:25.917 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:26.062 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] check diff failed 5-th time, retry later [2024/05/31 17:56:26.095 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:26.122 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:26.166 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:26.187 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:26.208 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:26.272 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:26.282 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:26.307 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:26.321 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:26.332 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:26.356 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:26.458 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:26.517 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:26.562 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsfee1aa74_f357_4b9b_b404_61ff4301d584"] + set +x [Fri May 31 17:56:26 CST 2024] <<<<<< START kafka consumer in multi_capture case >>>>>> table multi_capture_1.usertable not exists for 1-th check, retry later [2024/05/31 17:56:26.589 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs59791a0d_39e1_429d_908f_3c1a1d52188c"] [2024/05/31 17:56:26.682 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs72a6e491_0f16_4475_819f_d6962161410f"] [2024/05/31 17:56:26.711 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsec8b321d_4156_44ed_8a39_e9a1d1847de1"] [2024/05/31 17:56:26.753 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs6d74d41c_d502_49a0_8c31_720709f6252e"] [2024/05/31 17:56:26.762 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsdc9268cd_49d4_443a_bb3c_269ad11a4788"] [2024/05/31 17:56:26.804 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsd97308bf_3e6d_46a1_9957_7a77dd8bfba3"] [2024/05/31 17:56:26.889 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:26.898 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:26.910 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:26.956 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:27.055 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs8eb9f064_90e5_4fd3_937a_a9924a8d05f9"] [2024/05/31 17:56:27.079 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:27.111 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:27.159 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:27.173 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:27.189 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:27.198 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:27.201 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:27.296 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:27.303 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:27.373 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:27.394 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:27.462 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:27.463 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:27.481 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:27.564 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:27.576 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:27.580 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:27.609 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:27.670 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:27.684 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:27.691 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:27.692 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:27.758 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:27.784 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:27.809 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] check diff successfully [2024/05/31 17:56:27.875 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:27.877 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:27.955 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:27.976 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:27.991 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:28.058 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:28.073 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:28.088 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:28.099 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:28.176 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:28.185 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:28.185 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:28.201 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:28.262 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:28.280 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:28.305 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5690.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:56:28.453 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:28.458 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:28.493 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:28.556 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:28.577 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:28.593 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:28.611 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] table multi_capture_1.usertable not exists for 2-th check, retry later table region_merge.t1 exists check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:56:28.668 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:28.668 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:28.708 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:28.712 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:28.758 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:28.782 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:28.783 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:28.790 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:28.795 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:28.898 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:28.902 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] check diff failed 1-th time, retry later [2024/05/31 17:56:28.984 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:29.057 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:29.073 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:29.089 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:29.105 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:29.165 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:29.167 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:29.259 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:29.261 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:29.274 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:29.287 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:29.306 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:29.358 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:29.365 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:29.456 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:29.466 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:29.495 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:29.576 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:29.598 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:29.655 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:29.676 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] + set +x + tso='450140409214271489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140409214271489 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.db"="changefeed_error" "threadcount"="4" "requestdistribution"="uniform" "readproportion"="0" "mysql.user"="root" "scanproportion"="0" "updateproportion"="0" "insertproportion"="0" "operationcount"="0" "recordcount"="20" "readallfields"="true" "dotransactions"="false" "workload"="core" ********************************************** Run finished, takes 9.094724ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3634.7, Avg(us): 1653, Min(us): 934, Max(us): 3552, 95th(us): 4000, 99th(us): 4000 [Fri May 31 17:56:29 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57635765.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:56:29.689 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:29.690 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:29.770 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:29.771 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:29.781 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:29.798 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:29.863 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:29.871 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:29.871 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:29.913 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:29.967 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:29.982 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:30.060 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:30.086 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:30.101 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:30.156 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:30.167 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:30.185 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:30.212 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:30.216 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:30.276 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:30.278 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:30.369 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:30.374 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:30.381 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] check diff successfully [2024/05/31 17:56:30.458 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:30.490 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:30.578 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:30.593 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:30.605 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:30.680 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:30.689 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:30.691 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:30.711 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] Verifying downstream PD is started... [2024/05/31 17:56:30.792 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:30.796 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:30.799 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:30.875 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:30.912 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] table multi_capture_1.usertable exists table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later [2024/05/31 17:56:31.016 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:31.019 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:56:31.065 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs52dc12db_9572_46a6_90f3_5c900a14c52e"] [2024/05/31 17:56:31.092 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs18f2c31a_75eb_4a7b_8e89_0b44e27a8767"] [2024/05/31 17:56:31.140 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLscda37535_07b6_4aaa_97bf_6fc92e64dfef"] [2024/05/31 17:56:31.208 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs34841e46_322b_4c57_97b6_127687d7807d"] wait process cdc.test exit for 1-th time... check diff failed 2-th time, retry later [2024/05/31 17:56:31.240 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsadd86edd_f140_4246_a374_dc3117bb8b0e"] [2024/05/31 17:56:31.283 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs3486c1e2_eeae_457c_9382_ded8ac368f8a"] [2024/05/31 17:56:31.454 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:31.482 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:31.482 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [2024/05/31 17:56:31.508 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:31.561 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:31.573 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsafcf11c1_76b2_4b40_a6ce_a4988ec6e3a4"] [2024/05/31 17:56:31.582 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:31.618 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:31.661 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:31.665 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:31.692 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:31.713 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsb1504be8_3a7d_48d0_90d2_61925aba7c3e"] wait process cdc.test exit for 2-th time... [2024/05/31 17:56:31.757 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:31.770 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:31.859 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:31.891 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:31.904 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:31.985 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:31.998 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:32.062 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:32.082 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:32.085 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:32.100 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:32.157 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:32.188 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:32.189 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:32.205 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:32.278 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:32.279 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:32.295 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:32.309 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:32.386 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:32.467 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:32.492 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:32.500 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] wait process cdc.test exit for 3-th time... [2024/05/31 17:56:32.560 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:32.595 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:32.596 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:32.596 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:32.600 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:32.679 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:32.687 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:32.698 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:32 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 {"id":"fcda3688-cf1f-4a76-8462-e7292b7f00e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149390} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14238fad fcda3688-cf1f-4a76-8462-e7292b7f00e6 /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 {"id":"fcda3688-cf1f-4a76-8462-e7292b7f00e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149390} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14238fad fcda3688-cf1f-4a76-8462-e7292b7f00e6 /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 {"id":"fcda3688-cf1f-4a76-8462-e7292b7f00e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149390} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14238fad fcda3688-cf1f-4a76-8462-e7292b7f00e6 /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5811.out cli changefeed create --start-ts=450140409214271489 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error [2024/05/31 17:56:32.773 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:32.774 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:32.789 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:32.796 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:32.869 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:32.957 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:32.968 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:33.000 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:33.006 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri May 31 17:56:33 CST 2024] <<<<<< run test case region_merge success! >>>>>> Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7375100422790484190,"namespace":"default","id":"changefeed-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:56:33.233261637+08:00","start_ts":450140409214271489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140409214271489,"checkpoint_ts":450140409214271489,"checkpoint_time":"2024-05-31 17:56:28.177"} PASS [2024/05/31 17:56:33.084 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:33.089 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:33.107 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:33.112 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:33.164 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:33.184 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:33.201 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] check diff failed 3-th time, retry later coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/31 17:56:33.293 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:33.294 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:33.297 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:33.305 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:33.366 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:33.499 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:33.503 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:33.559 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:33.594 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:33.611 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:33.658 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:33.688 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:33.694 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:33.706 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:33.759 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:33.774 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:33.862 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:33.863 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:33.868 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:33.870 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:33.870 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:33.993 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:34.004 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:34.053 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:56:34.102 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:34.108 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:34.113 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:34.191 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:34.196 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:34.198 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:34.208 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:34.269 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:34.316 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:34.356 +08:00] [INFO] [main.go:178] ["72 insert success: 500"] [2024/05/31 17:56:34.361 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:34.364 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:34.367 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:34.480 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:34.489 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:34.502 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:34.597 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:34.599 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:34.602 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:34.689 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:34.701 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:34.701 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:34.702 +08:00] [INFO] [main.go:178] ["73 insert success: 600"] [2024/05/31 17:56:34.780 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:34.819 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] + set +x [Fri May 31 17:56:34 CST 2024] <<<<<< START kafka consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450140409214271489, "checkpoint_time": "2024-05-31 17:56:28.177", "error": { "time": "2024-05-31T17:56:33.385313631+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450140409214271189 is earlier than or equal to GC safepoint at 450140409214271489" } }' + echo '{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450140409214271489, "checkpoint_time": "2024-05-31 17:56:28.177", "error": { "time": "2024-05-31T17:56:33.385313631+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450140409214271189 is earlier than or equal to GC safepoint at 450140409214271489" } }' { "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 450140409214271489, "checkpoint_time": "2024-05-31 17:56:28.177", "error": { "time": "2024-05-31T17:56:33.385313631+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450140409214271189 is earlier than or equal to GC safepoint at 450140409214271489" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 450140409214271489, '"checkpoint_time":' '"2024-05-31' '17:56:28.177",' '"error":' '{' '"time":' '"2024-05-31T17:56:33.385313631+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 450140409214271189 is earlier than or equal to GC safepoint at '450140409214271489"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 450140409214271489, '"checkpoint_time":' '"2024-05-31' '17:56:28.177",' '"error":' '{' '"time":' '"2024-05-31T17:56:33.385313631+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 450140409214271189 is earlier than or equal to GC safepoint at '450140409214271489"' '}' '}' table multi_capture_4.usertable exists check diff failed 1-th time, retry later [2024/05/31 17:56:34.874 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:34.874 +08:00] [INFO] [main.go:178] ["72 insert success: 600"] [2024/05/31 17:56:34.879 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:34.919 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:34.971 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:35.020 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:35.023 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450140409214271189 is earlier than or equal to GC safepoint at 450140409214271489' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 450140409214271189 is earlier than or equal to GC safepoint at 450140409214271489 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5931.out cli changefeed resume -c changefeed-error check diff failed 4-th time, retry later [2024/05/31 17:56:35.092 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:35.112 +08:00] [INFO] [main.go:178] ["73 insert success: 700"] [2024/05/31 17:56:35.121 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:35.181 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:35.360 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:35.364 +08:00] [INFO] [main.go:178] ["72 insert success: 700"] [2024/05/31 17:56:35.367 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:35.553 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... [2024/05/31 17:56:35.654 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:56:35.696 +08:00] [INFO] [main.go:178] ["73 insert success: 900"] [2024/05/31 17:56:35.709 +08:00] [INFO] [main.go:178] ["73 insert success: 800"] [2024/05/31 17:56:35.732 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:35.767 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9de9e353_c77c_44a5_8ec7_60d3561d9ebe"] [2024/05/31 17:56:35.798 +08:00] [INFO] [main.go:178] ["72 insert success: 800"] [2024/05/31 17:56:35.886 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsee9d08e3_fe6d_429a_b407_4bb3565156c6"] [2024/05/31 17:56:35.970 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs45cbf0a7_c254_4df4_8ab4_e20a2d9e5075"] [2024/05/31 17:56:36.067 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLs9e1ea5ab_3d8a_4e2c_838b_29ba03cef86a"] [2024/05/31 17:56:36.074 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:36.076 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:56:36.141 +08:00] [INFO] [main.go:835] ["running ddl test: testMultiDDLsf385a23f_667a_4f2a_9cee_a7a4cc79e27e"] [2024/05/31 17:56:36.191 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:36.194 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:36.269 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:36.396 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:36.403 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:36.408 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:36.413 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:36.414 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:36.499 +08:00] [INFO] [main.go:178] ["73 insert success: 100"] [2024/05/31 17:56:36.505 +08:00] [INFO] [main.go:178] ["72 insert success: 100"] [2024/05/31 17:56:36.559 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:36.566 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:36.613 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:36.764 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] [2024/05/31 17:56:36.770 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:36.790 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:36.794 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:36.798 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:36.888 +08:00] [INFO] [main.go:178] ["73 insert success: 200"] + set +x table changefeed_error.usertable not exists for 1-th check, retry later [2024/05/31 17:56:36.896 +08:00] [INFO] [main.go:178] ["72 insert success: 200"] [2024/05/31 17:56:36.962 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:36.980 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:37.022 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] table test1.finishmark exists [2024/05/31 17:56:35.278 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/05/31 17:56:35.280 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/05/31 17:56:35.282 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/05/31 17:56:35.287 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/05/31 17:56:35.289 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/05/31 17:56:35.290 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=15.789561ms] [2024/05/31 17:56:35.294 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=3030946575] [2024/05/31 17:56:35.295 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t2] [checkSum=718014124] [2024/05/31 17:56:35.297 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t3] [checkSum=718014124] [2024/05/31 17:56:35.302 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.finishmark] [checkSum=0] [2024/05/31 17:56:35.304 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test1.t1] [checkSum=718014124] [2024/05/31 17:56:35.304 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=14.403611ms] [2024/05/31 17:56:35.304 +08:00] [INFO] [main.go:95] ["compare checksum passed"] wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:56:36 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> [2024/05/31 17:56:37.161 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:37.166 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:37.189 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:37.190 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:37.201 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:37.262 +08:00] [INFO] [main.go:178] ["73 insert success: 300"] [2024/05/31 17:56:37.271 +08:00] [INFO] [main.go:178] ["72 insert success: 300"] [2024/05/31 17:56:37.300 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:37.364 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:37.404 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] check diff successfully ***************** properties ***************** "scanproportion"="0" "updateproportion"="0" "recordcount"="20" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.user"="root" "operationcount"="0" "threadcount"="2" "readallfields"="true" "mysql.db"="multi_capture_1" "readproportion"="0" "insertproportion"="0" "dotransactions"="false" "requestdistribution"="uniform" "workload"="core" ********************************************** Run finished, takes 11.72055ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2003.0, Avg(us): 1126, Min(us): 465, Max(us): 1913, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "workload"="core" "updateproportion"="0" "threadcount"="2" "mysql.port"="4000" "insertproportion"="0" "mysql.db"="multi_capture_2" "mysql.user"="root" "requestdistribution"="uniform" "readallfields"="true" "readproportion"="0" "dotransactions"="false" "scanproportion"="0" "operationcount"="0" "recordcount"="20" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 14.115034ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1671.8, Avg(us): 1244, Min(us): 614, Max(us): 2689, 95th(us): 3000, 99th(us): 3000 ***************** properties ***************** "mysql.user"="root" "mysql.port"="4000" "updateproportion"="0" "dotransactions"="false" "mysql.db"="multi_capture_3" "readproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "workload"="core" "threadcount"="2" "readallfields"="true" "operationcount"="0" "insertproportion"="0" "recordcount"="20" "requestdistribution"="uniform" ********************************************** Run finished, takes 25.745725ms INSERT - Takes(s): 0.0, Count: 19, OPS: 1127.2, Avg(us): 2189, Min(us): 730, Max(us): 8756, 95th(us): 9000, 99th(us): 9000 ***************** properties ***************** "readproportion"="0" "workload"="core" "mysql.db"="multi_capture_4" "mysql.host"="127.0.0.1" "operationcount"="0" "dotransactions"="false" "mysql.port"="4000" "scanproportion"="0" "insertproportion"="0" "threadcount"="2" "mysql.user"="root" "updateproportion"="0" "readallfields"="true" "requestdistribution"="uniform" "recordcount"="20" ********************************************** Run finished, takes 13.022852ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1692.6, Avg(us): 1155, Min(us): 623, Max(us): 1852, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later [2024/05/31 17:56:37.584 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] [2024/05/31 17:56:37.635 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:37.658 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:37.662 +08:00] [INFO] [main.go:178] ["73 insert success: 400"] check diff failed 5-th time, retry later [2024/05/31 17:56:37.688 +08:00] [INFO] [main.go:178] ["72 insert success: 400"] [2024/05/31 17:56:37.696 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] [2024/05/31 17:56:37.752 +08:00] [INFO] [main.go:178] ["73 insert success: 500"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3851abc40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:14225, start at 2024-05-31 17:56:35.728774194 +0800 CST m=+2.037479370 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:35.737 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:35.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:35.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3851abc40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:14225, start at 2024-05-31 17:56:35.728774194 +0800 CST m=+2.037479370 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:35.737 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:35.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:35.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3851ac000005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:14291, start at 2024-05-31 17:56:35.715998442 +0800 CST m=+1.967114670 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:35.724 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:35.712 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:35.712 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/31 17:56:38.004 +08:00] [INFO] [main.go:812] ["testMultiDDLs take %v22.354779378s"] [2024/05/31 17:56:38.039 +08:00] [INFO] [main.go:74] ["DefaultValue integration tests take 22.389953886s"] Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table mark.finish_mark_1 not exists for 1-th check, retry later table changefeed_error.usertable exists check diff failed 1-th time, retry later check diff successfully check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore wait process cdc.test exit for 1-th time... Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later [Fri May 31 17:56:40 CST 2024] <<<<<< START cdc server in kafka_simple_handle_key_only_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.1562015622.out server --log-file /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_handle_key_only_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_1 not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:56:40 CST 2024] <<<<<< run test case multi_capture success! >>>>>> check diff successfully ***************** properties ***************** "updateproportion"="0" "recordcount"="20" "requestdistribution"="uniform" "mysql.user"="root" "insertproportion"="0" "threadcount"="4" "operationcount"="0" "dotransactions"="false" "readproportion"="0" "mysql.db"="changefeed_error" "mysql.host"="127.0.0.1" "workload"="core" "readallfields"="true" "mysql.port"="4000" "scanproportion"="0" ********************************************** Run finished, takes 4.786355ms INSERT - Takes(s): 0.0, Count: 20, OPS: 5774.1, Avg(us): 835, Min(us): 505, Max(us): 2010, 95th(us): 2000, 99th(us): 3000 check diff successfully {"id":"fcda3688-cf1f-4a76-8462-e7292b7f00e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149390} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 + [[ /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_1 not exists for 3-th check, retry later check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:43 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df85bc9b-0e26-48fc-bac2-44bc7fd26dfe {"id":"df85bc9b-0e26-48fc-bac2-44bc7fd26dfe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce144eff8b df85bc9b-0e26-48fc-bac2-44bc7fd26dfe /tidb/cdc/default/default/upstream/7375100470851344743 {"id":7375100470851344743,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df85bc9b-0e26-48fc-bac2-44bc7fd26dfe {"id":"df85bc9b-0e26-48fc-bac2-44bc7fd26dfe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce144eff8b df85bc9b-0e26-48fc-bac2-44bc7fd26dfe /tidb/cdc/default/default/upstream/7375100470851344743 {"id":7375100470851344743,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df85bc9b-0e26-48fc-bac2-44bc7fd26dfe {"id":"df85bc9b-0e26-48fc-bac2-44bc7fd26dfe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149400} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce144eff8b df85bc9b-0e26-48fc-bac2-44bc7fd26dfe /tidb/cdc/default/default/upstream/7375100470851344743 {"id":7375100470851344743,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.15675.out cli tso query --pd=http://127.0.0.1:2379 pass check, checkpoint tso not forward after 10s run task successfully wait process 8689 exit for 1-th time... wait process 8689 exit for 2-th time... wait process 8689 exit for 3-th time... wait process 8689 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8689) - No such process wait process 8689 exit for 5-th time... process 8689 already exit [Fri May 31 17:56:36 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.92639265.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:39 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7375100147348505938 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:55:26.411100565 +0800 CST StartTs:450140392982052867 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e18510 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140393021112321} {CheckpointTs:450140396783140881 MinTableBarrierTs:450140411974385667 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85eac2e5-03db-485d-9fcd-e9467956fec7 {"id":"85eac2e5-03db-485d-9fcd-e9467956fec7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149396} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce132ba10e 85eac2e5-03db-485d-9fcd-e9467956fec7 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375100147348505938,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:55:26.411100565+08:00","start-ts":450140392982052867,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140393021112321} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450140396783140881,"min-table-barrier-ts":450140411974385667,"admin-job-type":0} /tidb/cdc/default/default/task/position/85eac2e5-03db-485d-9fcd-e9467956fec7/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100147348505938 {"id":7375100147348505938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7375100147348505938 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:55:26.411100565 +0800 CST StartTs:450140392982052867 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e18510 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140393021112321} {CheckpointTs:450140396783140881 MinTableBarrierTs:450140411974385667 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85eac2e5-03db-485d-9fcd-e9467956fec7 {"id":"85eac2e5-03db-485d-9fcd-e9467956fec7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149396} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce132ba10e 85eac2e5-03db-485d-9fcd-e9467956fec7 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375100147348505938,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:55:26.411100565+08:00","start-ts":450140392982052867,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140393021112321} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450140396783140881,"min-table-barrier-ts":450140411974385667,"admin-job-type":0} /tidb/cdc/default/default/task/position/85eac2e5-03db-485d-9fcd-e9467956fec7/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100147348505938 {"id":7375100147348505938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7375100147348505938 Namespace:default ID:ddl-only-block-related-table SinkURI:kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:55:26.411100565 +0800 CST StartTs:450140392982052867 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e18510 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140393021112321} {CheckpointTs:450140396783140881 MinTableBarrierTs:450140411974385667 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 450140396783140881, checkpointTs: 450140396783140881, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/85eac2e5-03db-485d-9fcd-e9467956fec7 {"id":"85eac2e5-03db-485d-9fcd-e9467956fec7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149396} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce132ba10e 85eac2e5-03db-485d-9fcd-e9467956fec7 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7375100147348505938,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-21786?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:55:26.411100565+08:00","start-ts":450140392982052867,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140393021112321} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":450140396783140881,"min-table-barrier-ts":450140411974385667,"admin-job-type":0} /tidb/cdc/default/default/task/position/85eac2e5-03db-485d-9fcd-e9467956fec7/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100147348505938 {"id":7375100147348505938,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 1-th time, retry later check diff failed 2-th time, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 + [[ /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later check diff successfully check_ts_forward ddl-only-block-related-table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38520a500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:7009, start at 2024-05-31 17:56:41.75336587 +0800 CST m=+2.047207958 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:41.762 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:41.748 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:41.748 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38520a500006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:7009, start at 2024-05-31 17:56:41.75336587 +0800 CST m=+2.047207958 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:41.762 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:41.748 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:41.748 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38520b000015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:7086, start at 2024-05-31 17:56:41.836049065 +0800 CST m=+2.074427544 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:41.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:41.842 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:41.842 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_1 not exists for 4-th check, retry later check diff failed 3-th time, retry later + set +x + tso='450140413217210371 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140413217210371 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.15697.out cli changefeed create --start-ts=450140413217210371 '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-avro-30299?protocol=simple&encoding-format=avro' -c simple-handle-key-only-avro --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/conf/changefeed.toml =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... Create changefeed successfully! ID: simple-handle-key-only-avro Info: {"upstream_id":7375100470851344743,"namespace":"default","id":"simple-handle-key-only-avro","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-avro-30299?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-31T17:56:45.341957495+08:00","start_ts":450140413217210371,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140413217210371,"checkpoint_ts":450140413217210371,"checkpoint_time":"2024-05-31 17:56:43.447"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... changefeed is working normally rts: 450140413284843500->450140413547249660 checkpoint: 450140413284843500->450140413547249660 run task successfully wait process cdc.test exit for 1-th time... table mark.finish_mark_1 not exists for 5-th check, retry later check diff failed 4-th time, retry later wait process cdc.test exit for 2-th time... [Fri May 31 17:56:46 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.84778479.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:56:47 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 + [[ /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fcda3688-cf1f-4a76-8462-e7292b7f00e6 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later table test.finish_mark not exists for 1-th check, retry later table mark.finish_mark_1 not exists for 6-th check, retry later table test.finish_mark not exists for 2-th check, retry later check diff failed 5-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/common_1 Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:49 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00728148-ddef-40a6-b7aa-a5148003ec91 {"id":"00728148-ddef-40a6-b7aa-a5148003ec91","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149406} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1464adc2 00728148-ddef-40a6-b7aa-a5148003ec91 /tidb/cdc/default/default/upstream/7375100501838391131 {"id":7375100501838391131,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00728148-ddef-40a6-b7aa-a5148003ec91 {"id":"00728148-ddef-40a6-b7aa-a5148003ec91","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149406} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1464adc2 00728148-ddef-40a6-b7aa-a5148003ec91 /tidb/cdc/default/default/upstream/7375100501838391131 {"id":7375100501838391131,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/00728148-ddef-40a6-b7aa-a5148003ec91 {"id":"00728148-ddef-40a6-b7aa-a5148003ec91","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149406} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1464adc2 00728148-ddef-40a6-b7aa-a5148003ec91 /tidb/cdc/default/default/upstream/7375100501838391131 {"id":7375100501838391131,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.8536.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-batch-add-table-test-7193?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 6678c744-ed5b-45c9-891d-ba2435a69b76 Info: {"upstream_id":7375100501838391131,"namespace":"default","id":"6678c744-ed5b-45c9-891d-ba2435a69b76","sink_uri":"kafka://127.0.0.1:9092/ticdc-batch-add-table-test-7193?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:56:50.035196129+08:00","start_ts":450140414895194118,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140414895194118,"checkpoint_ts":450140414895194118,"checkpoint_time":"2024-05-31 17:56:49.848"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/31 17:56:47.505 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] [2024/05/31 17:56:47.836 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:56:47.852 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:56:48.006 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:56:48.008 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:56:48.035 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:56:48.037 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:56:48.186 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 17:56:48.226 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:56:49.398 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:56:49.403 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:56:49.447 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:56:49.449 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:56:49.585 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 17:56:49.639 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 17:56:49.777 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:56:49.783 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 17:56:49.825 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:56:49.827 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] table mark.finish_mark_1 not exists for 7-th check, retry later table test.finish_mark not exists for 3-th check, retry later Verifying downstream PD is started... [2024/05/31 17:56:50.990 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:56:51.104 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] check diff successfully [2024/05/31 17:56:51.170 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:56:51.179 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:56:51.293 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 17:56:51.295 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:56:51.359 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] + set +x [Fri May 31 17:56:51 CST 2024] <<<<<< START kafka consumer in batch_add_table case >>>>>> table batch_add_table.finish_mark not exists for 1-th check, retry later [2024/05/31 17:56:51.490 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh: line 1: 16863 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 (wd: /tmp/tidb_cdc_test/kafka_column_selector) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 check diff failed 1-th time, retry later [mysql] 2024/05/31 17:56:52 connection.go:299: invalid connection table mark.finish_mark_1 not exists for 8-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... table test.finish_mark not exists for 4-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.15760.out cli changefeed pause -c simple-handle-key-only-avro check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri May 31 17:56:53 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.61836185.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark not exists for 2-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 2-th time, retry later table mark.finish_mark_1 not exists for 9-th check, retry later table test.finish_mark not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.15791.out cli changefeed update -c simple-handle-key-only-avro '--sink-uri=kafka://127.0.0.1:9092/simple-handle-key-only-avro-30299?protocol=simple&encoding-format=avro&max-message-bytes=650' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/conf/changefeed.toml --no-confirm table batch_add_table.finish_mark exists check diff successfully * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:55 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 49 100 24 100 25 338 352 --:--:-- --:--:-- --:--:-- 357 {"compatibility":"NONE"}The 1 times to try to start tidb cluster... Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/simple-handle-key-only-avro-30299?protocol=simple&encoding-format=avro To:kafka://127.0.0.1:9092/simple-handle-key-only-avro-30299?protocol=simple&encoding-format=avro&max-message-bytes=650} {Type:update Path:[Config SyncPointInterval] From: To:0xc0035e58d8} {Type:update Path:[Config SyncPointRetention] From: To:0xc0035e58e8} {Type:update Path:[Config Consistent] From: To:0xc0010e1960} Update changefeed config successfully! ID: simple-handle-key-only-avro Info: {"upstream_id":7375100470851344743,"namespace":"default","id":"simple-handle-key-only-avro","sink_uri":"kafka://127.0.0.1:9092/simple-handle-key-only-avro-30299?protocol=simple\u0026encoding-format=avro\u0026max-message-bytes=650","create_time":"2024-05-31T17:56:45.341957495+08:00","start_ts":450140413217210371,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":0,"send_bootstrap_in_msg_count":0,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":0,"checkpoint_ts":450140415655411714,"checkpoint_time":"2024-05-31 17:56:52.748"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:56:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8c1ab18-71b9-46d0-9878-37487120d38e {"id":"e8c1ab18-71b9-46d0-9878-37487120d38e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149413} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fce14239091 e8c1ab18-71b9-46d0-9878-37487120d38e /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7375100422790484190,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:56:33.233261637+08:00","start-ts":450140409214271489,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-31T17:56:53.860733884+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140415938527236} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450140412465119235,"min-table-barrier-ts":450140412465119235,"admin-job-type":1} /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8c1ab18-71b9-46d0-9878-37487120d38e {"id":"e8c1ab18-71b9-46d0-9878-37487120d38e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149413} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fce14239091 e8c1ab18-71b9-46d0-9878-37487120d38e /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7375100422790484190,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:56:33.233261637+08:00","start-ts":450140409214271489,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-31T17:56:53.860733884+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140415938527236} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450140412465119235,"min-table-barrier-ts":450140412465119235,"admin-job-type":1} /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e8c1ab18-71b9-46d0-9878-37487120d38e {"id":"e8c1ab18-71b9-46d0-9878-37487120d38e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149413} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318fce14239091 e8c1ab18-71b9-46d0-9878-37487120d38e /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7375100422790484190,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:56:33.233261637+08:00","start-ts":450140409214271489,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-31T17:56:53.860733884+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140415938527236} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":450140412465119235,"min-table-barrier-ts":450140412465119235,"admin-job-type":1} /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s check diff failed 3-th time, retry later wait process cdc.test exit for 2-th time... + info='{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450140412465119235, "checkpoint_time": "2024-05-31 17:56:40.578", "error": { "time": "2024-05-31T17:56:53.860733884+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450140412465119235, "checkpoint_time": "2024-05-31 17:56:40.578", "error": { "time": "2024-05-31T17:56:53.860733884+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 450140412465119235, "checkpoint_time": "2024-05-31 17:56:40.578", "error": { "time": "2024-05-31T17:56:53.860733884+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 450140412465119235, '"checkpoint_time":' '"2024-05-31' '17:56:40.578",' '"error":' '{' '"time":' '"2024-05-31T17:56:53.860733884+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 450140412465119235, '"checkpoint_time":' '"2024-05-31' '17:56:40.578",' '"error":' '{' '"time":' '"2024-05-31T17:56:53.860733884+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6270.out cli changefeed remove -c changefeed-error table mark.finish_mark_1 not exists for 10-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3852f2d00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b, pid:6983, start at 2024-05-31 17:56:56.655078839 +0800 CST m=+1.698291218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:56.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:56.628 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:56.628 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3852f2d00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b, pid:6983, start at 2024-05-31 17:56:56.655078839 +0800 CST m=+1.698291218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:56.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:56.628 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:56.628 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3852f5dc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fv4zk-mhq5b, pid:7052, start at 2024-05-31 17:56:56.856644356 +0800 CST m=+1.845024231 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:58:56.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:56:56.823 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:46:56.823 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:56:57 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> Changefeed remove successfully. ID: changefeed-error CheckpointTs: 450140412465119235 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_handle_key_only_avro.cli.15814.out cli changefeed resume -c simple-handle-key-only-avro Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/common_1/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/common_1/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/common_1/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/common_1/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } coverage: 2.1% of statements in github.com/pingcap/tiflow/... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check_changefeed_state http://127.0.0.1:2379 0da8f6a4-f6e5-4dba-9126-a62706710aef finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=0da8f6a4-f6e5-4dba-9126-a62706710aef + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 0da8f6a4-f6e5-4dba-9126-a62706710aef -s + info='{ "upstream_id": 7375100087745084139, "namespace": "default", "id": "0da8f6a4-f6e5-4dba-9126-a62706710aef", "state": "finished", "checkpoint_tso": 450140413299261447, "checkpoint_time": "2024-05-31 17:56:43.760", "error": null }' + echo '{ "upstream_id": 7375100087745084139, "namespace": "default", "id": "0da8f6a4-f6e5-4dba-9126-a62706710aef", "state": "finished", "checkpoint_tso": 450140413299261447, "checkpoint_time": "2024-05-31 17:56:43.760", "error": null }' { "upstream_id": 7375100087745084139, "namespace": "default", "id": "0da8f6a4-f6e5-4dba-9126-a62706710aef", "state": "finished", "checkpoint_tso": 450140413299261447, "checkpoint_time": "2024-05-31 17:56:43.760", "error": null } ++ echo '{' '"upstream_id":' 7375100087745084139, '"namespace":' '"default",' '"id":' '"0da8f6a4-f6e5-4dba-9126-a62706710aef",' '"state":' '"finished",' '"checkpoint_tso":' 450140413299261447, '"checkpoint_time":' '"2024-05-31' '17:56:43.760",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7375100087745084139, '"namespace":' '"default",' '"id":' '"0da8f6a4-f6e5-4dba-9126-a62706710aef",' '"state":' '"finished",' '"checkpoint_tso":' 450140413299261447, '"checkpoint_time":' '"2024-05-31' '17:56:43.760",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri May 31 17:56:53 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> check diff failed 4-th time, retry later table mark.finish_mark_1 not exists for 11-th check, retry later + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully table test.finish_mark not exists for 7-th check, retry later [2024/05/31 17:56:58.021 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] [2024/05/31 17:56:58.245 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:56:58.270 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:56:58.427 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:56:58.429 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:56:58.482 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:56:58.483 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:56:59.078 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] start tidb cluster in /tmp/tidb_cdc_test/kafka_column_selector_avro Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x [2024/05/31 17:56:59.148 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 17:56:59.269 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:56:59.274 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:56:59.365 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:56:59.368 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:56:59.451 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 17:56:59.563 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/31 17:56:59.628 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:56:59.633 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:56:59.765 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:56:59.768 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 17:56:59.809 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] table test.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... [2024/05/31 17:56:59.969 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:56:59.999 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 17:57:00.008 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:57:00.158 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:57:00.165 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:57:00.177 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 17:57:00.347 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:57:00.348 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 17:57:00.352 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] check diff failed 5-th time, retry later table mark.finish_mark_1 not exists for 12-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri May 31 17:57:00 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.63596361.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:57:00.533 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 17:57:00.542 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 17:57:00.552 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] table test.finish_mark not exists for 8-th check, retry later [2024/05/31 17:57:00.704 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 17:57:00.709 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:57:00.725 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 17:57:00.882 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 17:57:00.912 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 17:57:00.923 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 17:57:01.062 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 17:57:01.072 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:57:01.105 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 17:57:01.237 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 17:57:01.282 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:57:01.292 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385336040019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:10453, start at 2024-05-31 17:57:00.970661656 +0800 CST m=+1.849387812 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:00.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:00.978 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:00.978 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385336040019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:10453, start at 2024-05-31 17:57:00.970661656 +0800 CST m=+1.849387812 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:00.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:00.978 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:00.978 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385336a80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:10523, start at 2024-05-31 17:57:01.010150917 +0800 CST m=+1.825750926 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:01.020 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:01.020 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:01.020 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.cli.8511.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:57:01.412 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 17:57:01.420 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 17:57:01.467 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 17:57:01.580 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 17:57:01.653 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 17:57:01.663 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 17:57:01.754 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 17:57:01.767 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 17:57:01.843 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 17:57:01.929 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 17:57:02.033 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 17:57:02.043 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 17:57:02.107 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/31 17:57:02.120 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] table test.finish_mark not exists for 2-th check, retry later [2024/05/31 17:57:02.224 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/31 17:57:02.305 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 17:57:02.424 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 17:57:02.434 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 17:57:02.493 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 17:57:02.504 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 17:57:02.611 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:57:02.661 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/31 17:57:02.794 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/31 17:57:02.805 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/31 17:57:02.901 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] check diff successfully table mark.finish_mark_1 not exists for 13-th check, retry later [2024/05/31 17:57:02.913 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/31 17:57:02.989 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] table test.finish_mark not exists for 9-th check, retry later check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:03 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96936ce2-a812-4cde-94cc-5896aecfcf1d {"id":"96936ce2-a812-4cde-94cc-5896aecfcf1d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149420} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fce142390eb 96936ce2-a812-4cde-94cc-5896aecfcf1d /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96936ce2-a812-4cde-94cc-5896aecfcf1d {"id":"96936ce2-a812-4cde-94cc-5896aecfcf1d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149420} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fce142390eb 96936ce2-a812-4cde-94cc-5896aecfcf1d /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/96936ce2-a812-4cde-94cc-5896aecfcf1d {"id":"96936ce2-a812-4cde-94cc-5896aecfcf1d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149420} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318fce142390eb 96936ce2-a812-4cde-94cc-5896aecfcf1d /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6405.out cli changefeed create --start-ts=450140409214271489 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-1 + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11892.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140418166751233 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140418166751233 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:57:03 CST 2024] <<<<<< START cdc server in common_1 case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.common_1.85438545.out server --log-file /tmp/tidb_cdc_test/common_1/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/common_1/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7375100422790484190,"namespace":"default","id":"changefeed-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:57:03.883548203+08:00","start_ts":450140409214271489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140409214271489,"checkpoint_ts":450140409214271489,"checkpoint_time":"2024-05-31 17:56:28.177"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark exists check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... table mark.finish_mark_1 not exists for 14-th check, retry later wait process cdc.test exit for 2-th time... [2024/05/31 17:56:57.626 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/31 17:56:57.626 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7375100056648633065] [2024/05/31 17:56:57.626 +08:00] [INFO] [client.go:613] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/31 17:56:57.626 +08:00] [INFO] [tso_client.go:293] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/31 17:56:57.627 +08:00] [INFO] [client.go:619] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/31 17:56:57.627 +08:00] [INFO] [tso_dispatcher.go:119] ["[tso] start tso deadline watcher"] [dc-location=global] [2024/05/31 17:56:57.627 +08:00] [INFO] [tso_client.go:130] ["[tso] start tso dispatcher check loop"] [2024/05/31 17:56:57.627 +08:00] [INFO] [tso_dispatcher.go:168] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/31 17:56:57.627 +08:00] [INFO] [tso_dispatcher.go:336] ["[tso] start tso connection contexts updater"] [dc-location=global] [2024/05/31 17:56:57.628 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/31 17:56:57.628 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7375100056648633065] [2024/05/31 17:56:57.628 +08:00] [INFO] [client.go:613] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/31 17:56:57.628 +08:00] [INFO] [tso_client.go:293] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/31 17:56:57.629 +08:00] [INFO] [client.go:619] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/31 17:56:57.629 +08:00] [INFO] [tso_dispatcher.go:168] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/31 17:56:57.629 +08:00] [INFO] [tso_client.go:130] ["[tso] start tso dispatcher check loop"] [2024/05/31 17:56:57.629 +08:00] [INFO] [tso_dispatcher.go:336] ["[tso] start tso connection contexts updater"] [dc-location=global] [2024/05/31 17:56:57.629 +08:00] [INFO] [tso_dispatcher.go:119] ["[tso] start tso deadline watcher"] [dc-location=global] [2024/05/31 17:56:57.629 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/31 17:56:57.630 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/31 17:56:57.631 +08:00] [INFO] [store_cache.go:510] ["change store resolve state"] [store=2] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] table test.finish_mark not exists for 10-th check, retry later + set +x + tso='450140418547122177 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140418547122177 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140418547122177 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri May 31 17:57:05 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1191611918.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1375 0 --:--:-- --:--:-- --:--:-- 1387 + info='{"state":"normal","resolved_ts":450140409214271489,"checkpoint_ts":450140409214271489}' + echo '{"state":"normal","resolved_ts":450140409214271489,"checkpoint_ts":450140409214271489}' {"state":"normal","resolved_ts":450140409214271489,"checkpoint_ts":450140409214271489} ++ echo '{"state":"normal","resolved_ts":450140409214271489,"checkpoint_ts":450140409214271489}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:57:05 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_1 not exists for 15-th check, retry later table test.finish_mark not exists for 11-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:06 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/281766cd-411f-4226-9063-921fb61ffebf {"id":"281766cd-411f-4226-9063-921fb61ffebf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149424} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce149e379a 281766cd-411f-4226-9063-921fb61ffebf /tidb/cdc/default/default/upstream/7375100556410499608 {"id":7375100556410499608,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/281766cd-411f-4226-9063-921fb61ffebf {"id":"281766cd-411f-4226-9063-921fb61ffebf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149424} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce149e379a 281766cd-411f-4226-9063-921fb61ffebf /tidb/cdc/default/default/upstream/7375100556410499608 {"id":7375100556410499608,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/281766cd-411f-4226-9063-921fb61ffebf {"id":"281766cd-411f-4226-9063-921fb61ffebf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149424} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce149e379a 281766cd-411f-4226-9063-921fb61ffebf /tidb/cdc/default/default/upstream/7375100556410499608 {"id":7375100556410499608,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: e5adad87-b966-4199-8663-f9a7b37775ba Info: {"upstream_id":7375100556410499608,"namespace":"default","id":"e5adad87-b966-4199-8663-f9a7b37775ba","sink_uri":"kafka://127.0.0.1:9092/ticdc-common-1-test-20568?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:57:07.051244355+08:00","start_ts":450140418166751233,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140418166751233,"checkpoint_ts":450140418166751233,"checkpoint_time":"2024-05-31 17:57:02.328"} [Fri May 31 17:57:07 CST 2024] <<<<<< START kafka consumer in common_1 case >>>>>> [2024/05/31 17:57:07.640 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/31 17:57:07.641 +08:00] [INFO] [tso_client.go:145] ["[tso] exit tso dispatcher check loop"] [2024/05/31 17:57:07.641 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/31 17:57:07.640 +08:00] [INFO] [tso_dispatcher.go:137] ["[tso] exit tso deadline watcher"] [dc-location=global] [2024/05/31 17:57:07.641 +08:00] [INFO] [tso_dispatcher.go:350] ["[tso] exit tso connection contexts updater"] [dc-location=global] [2024/05/31 17:57:07.641 +08:00] [INFO] [tso_dispatcher.go:210] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/31 17:57:07.641 +08:00] [INFO] [tso_dispatcher.go:171] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/31 17:57:07.641 +08:00] [INFO] [tso_batch_controller.go:160] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/31 17:57:07.641 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/31 17:57:07.641 +08:00] [INFO] [tso_client.go:155] ["[tso] closing tso client"] [2024/05/31 17:57:07.641 +08:00] [INFO] [tso_client.go:160] ["[tso] close tso client"] [2024/05/31 17:57:07.641 +08:00] [INFO] [tso_batch_controller.go:160] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/31 17:57:07.641 +08:00] [INFO] [tso_client.go:162] ["[tso] tso client is closed"] [2024/05/31 17:57:07.641 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38539ad4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:30388, start at 2024-05-31 17:57:07.392792568 +0800 CST m=+1.806625474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:07.399 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:07.381 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:07.381 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38539ad4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:30388, start at 2024-05-31 17:57:07.392792568 +0800 CST m=+1.806625474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:07.399 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:07.381 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:07.381 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38539b84000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:30471, start at 2024-05-31 17:57:07.437544178 +0800 CST m=+1.786787981 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:07.445 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:07.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:07.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table common_1.v1 not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_column_selector_avro/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 3-th time, retry later check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 252 100 252 0 0 4201 0 --:--:-- --:--:-- --:--:-- 4271 + info='{"state":"warning","resolved_ts":450140409581273118,"checkpoint_ts":450140409581273118,"last_warning":{"time":"2024-05-31T17:57:05.81013071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":450140409581273118,"checkpoint_ts":450140409581273118,"last_warning":{"time":"2024-05-31T17:57:05.81013071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":450140409581273118,"checkpoint_ts":450140409581273118,"last_warning":{"time":"2024-05-31T17:57:05.81013071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":450140409581273118,"checkpoint_ts":450140409581273118,"last_warning":{"time":"2024-05-31T17:57:05.81013071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ echo '{"state":"warning","resolved_ts":450140409581273118,"checkpoint_ts":450140409581273118,"last_warning":{"time":"2024-05-31T17:57:05.81013071+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .last_warning.message + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6470.out cli changefeed remove -c changefeed-error-1 Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 450140409581273118 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:08 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfd91a1c-3e05-4aa4-9aca-b4ee9701a9a8 {"id":"cfd91a1c-3e05-4aa4-9aca-b4ee9701a9a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14ba577e cfd91a1c-3e05-4aa4-9aca-b4ee9701a9a8 /tidb/cdc/default/default/upstream/7375100585807159605 {"id":7375100585807159605,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfd91a1c-3e05-4aa4-9aca-b4ee9701a9a8 {"id":"cfd91a1c-3e05-4aa4-9aca-b4ee9701a9a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14ba577e cfd91a1c-3e05-4aa4-9aca-b4ee9701a9a8 /tidb/cdc/default/default/upstream/7375100585807159605 {"id":7375100585807159605,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfd91a1c-3e05-4aa4-9aca-b4ee9701a9a8 {"id":"cfd91a1c-3e05-4aa4-9aca-b4ee9701a9a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14ba577e cfd91a1c-3e05-4aa4-9aca-b4ee9701a9a8 /tidb/cdc/default/default/upstream/7375100585807159605 {"id":7375100585807159605,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140418547122177 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11970.out cli changefeed create --start-ts=450140418547122177 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml table mark.finish_mark_1 not exists for 16-th check, retry later Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375100585807159605,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T17:57:08.769253963+08:00","start_ts":450140418547122177,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140418547122177,"checkpoint_ts":450140418547122177,"checkpoint_time":"2024-05-31 17:57:03.779"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 12-th check, retry later [2024/05/31 17:57:08.936 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/05/31 17:57:09.198 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:57:09.213 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:57:09.408 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:57:09.409 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:57:09.441 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:57:09.444 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:57:09.623 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:57:09.679 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] + set +x check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3853c1a40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:10536, start at 2024-05-31 17:57:09.886051634 +0800 CST m=+1.921431547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:09.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:09.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:09.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed [2024/05/31 17:57:10.135 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:57:10.136 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:57:10.210 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:57:10.212 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3853c1a40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:10536, start at 2024-05-31 17:57:09.886051634 +0800 CST m=+1.921431547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:09.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:09.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:09.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3853c0b40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-m7t4m-d0k17, pid:10601, start at 2024-05-31 17:57:09.843307082 +0800 CST m=+1.829670117 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:09.850 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:09.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:09.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table common_1.v1 not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 4635 0 --:--:-- --:--:-- --:--:-- 4702 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 17:57:03.779","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-31 17:57:10.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:57:03.779","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:57:10.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:57:03.779","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:57:10.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-31 17:57:03.779' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:57:03.779","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:57:10.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:57:03.779","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '17:57:10.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-31 17:57:10' + echo 'sink_checkpoint_ts is 2024-05-31' 17:57:03.779 sink_checkpoint_ts is 2024-05-31 17:57:03.779 ++ date -d '2024-05-31 17:57:03.779' +%s + checkpoint_timestamp=1717149423 ++ date -d '2024-05-31 17:57:10' +%s + current_timestamp=1717149430 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' [Fri May 31 17:57:09 CST 2024] <<<<<< START cdc server in kafka_column_selector_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.3187631878.out server --log-file /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_column_selector_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:57:10.360 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 17:57:10.432 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] table mark.finish_mark_1 exists table mark.finish_mark_2 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... [2024/05/31 17:57:10.596 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:57:10.598 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:57:10.679 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:57:10.682 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] table test.finish_mark not exists for 13-th check, retry later [2024/05/31 17:57:10.820 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 17:57:10.909 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:57:11.019 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:57:11.020 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Fri May 31 17:57:11 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.65196521.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:57:11.121 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:57:11.123 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:57:11.214 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 17:57:11.329 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 17:57:11.413 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 17:57:11.414 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:57:11.546 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 17:57:11.549 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 17:57:11.611 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 17:57:11.776 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 17:57:11.801 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:57:11.802 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 17:57:11.973 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 17:57:11.976 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 17:57:11.994 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] table common_1.v1 not exists for 3-th check, retry later [2024/05/31 17:57:12.187 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 17:57:12.198 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:57:12.199 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] check diff failed 5-th time, retry later [2024/05/31 17:57:12.379 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 17:57:12.389 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:57:12.391 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] table mark.finish_mark_2 not exists for 2-th check, retry later [2024/05/31 17:57:12.577 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 17:57:12.578 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 17:57:12.606 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 17:57:12.765 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 17:57:12.813 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 17:57:12.815 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 17:57:12.942 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 17:57:12.943 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 17:57:13.020 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 17:57:13.121 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 17:57:13.230 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 17:57:13.232 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 17:57:13.300 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 17:57:13.300 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] table test.finish_mark not exists for 14-th check, retry later ***************** properties ***************** "scanproportion"="0" "mysql.user"="root" "mysql.host"="127.0.0.1" "workload"="core" "readproportion"="0" "readallfields"="true" "mysql.db"="changefeed_auto_stop_1" "threadcount"="4" "recordcount"="20" "updateproportion"="0" "insertproportion"="0" "operationcount"="0" "requestdistribution"="uniform" "dotransactions"="false" "mysql.port"="4000" ********************************************** Run finished, takes 10.471459ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3736.9, Avg(us): 2012, Min(us): 1113, Max(us): 5032, 95th(us): 6000, 99th(us): 6000 table test.t1 exists + sleep 5 [2024/05/31 17:57:13.433 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/31 17:57:13.488 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:13 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08fdd4bf-eb94-4255-a969-b0308757abc9 {"id":"08fdd4bf-eb94-4255-a969-b0308757abc9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14c6d97d 08fdd4bf-eb94-4255-a969-b0308757abc9 /tidb/cdc/default/default/upstream/7375100610659643718 {"id":7375100610659643718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08fdd4bf-eb94-4255-a969-b0308757abc9 {"id":"08fdd4bf-eb94-4255-a969-b0308757abc9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14c6d97d 08fdd4bf-eb94-4255-a969-b0308757abc9 /tidb/cdc/default/default/upstream/7375100610659643718 {"id":7375100610659643718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/08fdd4bf-eb94-4255-a969-b0308757abc9 {"id":"08fdd4bf-eb94-4255-a969-b0308757abc9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149430} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14c6d97d 08fdd4bf-eb94-4255-a969-b0308757abc9 /tidb/cdc/default/default/upstream/7375100610659643718 {"id":7375100610659643718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_column_selector_avro.cli.31933.out cli changefeed create --start-ts=450140420172939265 '--sink-uri=kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true' -c test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 Create changefeed successfully! ID: test Info: {"upstream_id":7375100610659643718,"namespace":"default","id":"test","sink_uri":"kafka://127.0.0.1:9092/column-selector-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true","create_time":"2024-05-31T17:57:13.530764114+08:00","start_ts":450140420172939265,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"column_selectors":[{"matcher":["test.*"],"columns":["*","!b"]}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140420172939265,"checkpoint_ts":450140420172939265,"checkpoint_time":"2024-05-31 17:57:09.981"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/31 17:57:13.632 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 17:57:13.635 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 17:57:13.665 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 17:57:13.666 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 17:57:13.828 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 17:57:13.857 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] table common_1.v1 exists table common_1.recover_and_insert not exists for 1-th check, retry later table test.t2 not exists for 1-th check, retry later check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:14 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd99e235-070c-4c68-97bf-568ae161e674 {"id":"cd99e235-070c-4c68-97bf-568ae161e674","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149431} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fce1423916c cd99e235-070c-4c68-97bf-568ae161e674 /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd99e235-070c-4c68-97bf-568ae161e674 {"id":"cd99e235-070c-4c68-97bf-568ae161e674","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149431} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fce1423916c cd99e235-070c-4c68-97bf-568ae161e674 /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cd99e235-070c-4c68-97bf-568ae161e674 {"id":"cd99e235-070c-4c68-97bf-568ae161e674","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149431} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318fce1423916c cd99e235-070c-4c68-97bf-568ae161e674 /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6563.out cli changefeed create --start-ts=450140409214271489 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-error-2 start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7375100422790484190,"namespace":"default","id":"changefeed-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:57:14.743867313+08:00","start_ts":450140409214271489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140409214271489,"checkpoint_ts":450140409214271489,"checkpoint_time":"2024-05-31 17:56:28.177"} PASS ***************** properties ***************** "mysql.db"="changefeed_auto_stop_2" "threadcount"="4" "workload"="core" "updateproportion"="0" "mysql.port"="4000" "insertproportion"="0" "recordcount"="20" "readproportion"="0" "operationcount"="0" "dotransactions"="false" "mysql.user"="root" "readallfields"="true" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "scanproportion"="0" ********************************************** Run finished, takes 10.357945ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3445.0, Avg(us): 1914, Min(us): 1051, Max(us): 4562, 95th(us): 5000, 99th(us): 5000 coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x [Fri May 31 17:57:14 CST 2024] <<<<<< START kafka consumer in kafka_column_selector_avro case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_column_selector_avro/conf/changefeed.toml schema registry uri found: http://127.0.0.1:8088 Starting build checksum checker... table test.finishmark not exists for 1-th check, retry later table mark.finish_mark_2 not exists for 3-th check, retry later table test.t2 exists check diff failed 1-th time, retry later table test.finish_mark not exists for 15-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... check diff successfully ***************** properties ***************** "threadcount"="4" "operationcount"="0" "workload"="core" "mysql.user"="root" "updateproportion"="0" "readproportion"="0" "dotransactions"="false" "readallfields"="true" "mysql.port"="4000" "mysql.db"="changefeed_auto_stop_3" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "recordcount"="20" "insertproportion"="0" "scanproportion"="0" ********************************************** Run finished, takes 21.002727ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1217.9, Avg(us): 2547, Min(us): 1161, Max(us): 12426, 95th(us): 5000, 99th(us): 13000 wait process cdc.test exit for 1-th time... table common_1.recover_and_insert not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 450140409214271489, "checkpoint_time": "2024-05-31 17:56:28.177", "error": { "time": "2024-05-31T17:57:14.821028154+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450140409214271489 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 450140409214271489, "checkpoint_time": "2024-05-31 17:56:28.177", "error": { "time": "2024-05-31T17:57:14.821028154+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450140409214271489 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 450140409214271489, "checkpoint_time": "2024-05-31 17:56:28.177", "error": { "time": "2024-05-31T17:57:14.821028154+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450140409214271489 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450140409214271489, '"checkpoint_time":' '"2024-05-31' '17:56:28.177",' '"error":' '{' '"time":' '"2024-05-31T17:57:14.821028154+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450140409214271489 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .state + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450140409214271489, '"checkpoint_time":' '"2024-05-31' '17:56:28.177",' '"error":' '{' '"time":' '"2024-05-31T17:57:14.821028154+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450140409214271489 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' wait process cdc.test exit for 2-th time... + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450140409214271489 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 450140409214271489 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6635.out cli changefeed remove -c changefeed-error-2 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:57:16 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> ***************** properties ***************** "dotransactions"="false" "mysql.user"="root" "requestdistribution"="uniform" "scanproportion"="0" "readallfields"="true" "updateproportion"="0" "recordcount"="20" "mysql.db"="changefeed_auto_stop_4" "workload"="core" "mysql.port"="4000" "threadcount"="4" "readproportion"="0" "mysql.host"="127.0.0.1" "insertproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 9.364867ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3855.5, Avg(us): 1749, Min(us): 921, Max(us): 4122, 95th(us): 5000, 99th(us): 5000 [Fri May 31 17:57:16 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1218812190.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_2 not exists for 4-th check, retry later Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 450140409214271489 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table test.finish_mark not exists for 16-th check, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... table test.finishmark not exists for 2-th check, retry later check diff failed 2-th time, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh: line 1: 15834 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table common_1.recover_and_insert not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 4460 0 --:--:-- --:--:-- --:--:-- 4500 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 17:57:16.178","puller_resolved_ts":"2024-05-31 17:57:10.429","last_synced_ts":"2024-05-31 17:57:10.879","now_ts":"2024-05-31 17:57:17.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:57:16.178","puller_resolved_ts":"2024-05-31' '17:57:10.429","last_synced_ts":"2024-05-31' '17:57:10.879","now_ts":"2024-05-31' '17:57:17.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '17:57:16.178","puller_resolved_ts":"2024-05-31' '17:57:10.429","last_synced_ts":"2024-05-31' '17:57:10.879","now_ts":"2024-05-31' '17:57:17.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + set +x table mark.finish_mark_2 not exists for 5-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... table test.finishmark exists [2024/05/31 17:57:19.165 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/05/31 17:57:19.167 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/05/31 17:57:19.167 +08:00] [INFO] [main.go:107] ["get checksum for the upstream success"] [elapsed=8.461429ms] [2024/05/31 17:57:19.173 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.finishmark] [checkSum=0] [2024/05/31 17:57:19.175 +08:00] [INFO] [main.go:186] ["do checkSum success"] [table=test.t1] [checkSum=718014124] [2024/05/31 17:57:19.175 +08:00] [INFO] [main.go:116] ["get checksum for the downstream success"] [elapsed=8.321815ms] [2024/05/31 17:57:19.175 +08:00] [INFO] [main.go:95] ["compare checksum passed"] wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... check diff failed 3-th time, retry later Verifying downstream PD is started... wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:19 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54cf56d8-c9f8-4042-94be-9a7c7da91f55 {"id":"54cf56d8-c9f8-4042-94be-9a7c7da91f55","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149437} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14d318fc 54cf56d8-c9f8-4042-94be-9a7c7da91f55 /tidb/cdc/default/default/upstream/7375100620834952792 {"id":7375100620834952792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54cf56d8-c9f8-4042-94be-9a7c7da91f55 {"id":"54cf56d8-c9f8-4042-94be-9a7c7da91f55","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149437} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14d318fc 54cf56d8-c9f8-4042-94be-9a7c7da91f55 /tidb/cdc/default/default/upstream/7375100620834952792 {"id":7375100620834952792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54cf56d8-c9f8-4042-94be-9a7c7da91f55 {"id":"54cf56d8-c9f8-4042-94be-9a7c7da91f55","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149437} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14d318fc 54cf56d8-c9f8-4042-94be-9a7c7da91f55 /tidb/cdc/default/default/upstream/7375100620834952792 {"id":7375100620834952792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:57:19 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1223012232.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_claim_check Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... table common_1.recover_and_insert exists table common_1.finish_mark not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:57:20 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> [2024/05/31 17:57:19.900 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/05/31 17:57:20.108 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:57:20.115 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:57:20.292 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:57:20.293 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:57:20.307 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:57:20.308 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:57:20.498 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:57:20.520 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri May 31 17:57:20 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.66776679.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_2 exists table mark.finish_mark_3 not exists for 1-th check, retry later [2024/05/31 17:57:20.673 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:57:20.674 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:57:20.717 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:57:20.718 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:57:20.848 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:57:20 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> [2024/05/31 17:57:20.901 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 17:57:21.017 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:57:21.018 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used check diff failed 4-th time, retry later [2024/05/31 17:57:21.607 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:57:21.609 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 17:57:21.724 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 17:57:21.795 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:57:21.913 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:57:21.914 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:57:21.984 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:57:21.986 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:57:22.107 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] table common_1.finish_mark not exists for 2-th check, retry later [2024/05/31 17:57:22.176 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 17:57:22.294 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 17:57:22.295 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:57:22.375 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 17:57:22.378 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 17:57:22.519 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 17:57:22.590 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] table mark.finish_mark_3 not exists for 2-th check, retry later [2024/05/31 17:57:22.715 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:57:22.716 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 17:57:22.795 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 17:57:22.797 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 17:57:22.907 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 17:57:23.001 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 17:57:23.128 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:57:23.129 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:22 GMT < Content-Length: 1271 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50246ca6-0563-4355-93d0-ac69d639c3b4 {"id":"50246ca6-0563-4355-93d0-ac69d639c3b4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149440} /tidb/cdc/default/__cdc_meta__/capture/54cf56d8-c9f8-4042-94be-9a7c7da91f55 {"id":"54cf56d8-c9f8-4042-94be-9a7c7da91f55","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149437} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14d318fc 54cf56d8-c9f8-4042-94be-9a7c7da91f55 /tidb/cdc/default/__cdc_meta__/owner/22318fce14d3192f 50246ca6-0563-4355-93d0-ac69d639c3b4 /tidb/cdc/default/default/upstream/7375100620834952792 {"id":7375100620834952792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50246ca6-0563-4355-93d0-ac69d639c3b4 {"id":"50246ca6-0563-4355-93d0-ac69d639c3b4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149440} /tidb/cdc/default/__cdc_meta__/capture/54cf56d8-c9f8-4042-94be-9a7c7da91f55 {"id":"54cf56d8-c9f8-4042-94be-9a7c7da91f55","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149437} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14d318fc 54cf56d8-c9f8-4042-94be-9a7c7da91f55 /tidb/cdc/default/__cdc_meta__/owner/22318fce14d3192f 50246ca6-0563-4355-93d0-ac69d639c3b4 /tidb/cdc/default/default/upstream/7375100620834952792 {"id":7375100620834952792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50246ca6-0563-4355-93d0-ac69d639c3b4 {"id":"50246ca6-0563-4355-93d0-ac69d639c3b4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149440} /tidb/cdc/default/__cdc_meta__/capture/54cf56d8-c9f8-4042-94be-9a7c7da91f55 {"id":"54cf56d8-c9f8-4042-94be-9a7c7da91f55","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149437} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14d318fc 54cf56d8-c9f8-4042-94be-9a7c7da91f55 /tidb/cdc/default/__cdc_meta__/owner/22318fce14d3192f 50246ca6-0563-4355-93d0-ac69d639c3b4 /tidb/cdc/default/default/upstream/7375100620834952792 {"id":7375100620834952792,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Fri May 31 17:57:23 CST 2024] <<<<<< START kafka consumer in changefeed_auto_stop case >>>>>> check_changefeed_state http://127.0.0.1:2379 939b98cd-a251-46b2-a26f-5b057f34e9bb normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=939b98cd-a251-46b2-a26f-5b057f34e9bb + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 939b98cd-a251-46b2-a26f-5b057f34e9bb -s + info='{ "upstream_id": 7375100620834952792, "namespace": "default", "id": "939b98cd-a251-46b2-a26f-5b057f34e9bb", "state": "normal", "checkpoint_tso": 450140420784783361, "checkpoint_time": "2024-05-31 17:57:12.315", "error": null }' + echo '{ "upstream_id": 7375100620834952792, "namespace": "default", "id": "939b98cd-a251-46b2-a26f-5b057f34e9bb", "state": "normal", "checkpoint_tso": 450140420784783361, "checkpoint_time": "2024-05-31 17:57:12.315", "error": null }' { "upstream_id": 7375100620834952792, "namespace": "default", "id": "939b98cd-a251-46b2-a26f-5b057f34e9bb", "state": "normal", "checkpoint_tso": 450140420784783361, "checkpoint_time": "2024-05-31 17:57:12.315", "error": null } ++ echo '{' '"upstream_id":' 7375100620834952792, '"namespace":' '"default",' '"id":' '"939b98cd-a251-46b2-a26f-5b057f34e9bb",' '"state":' '"normal",' '"checkpoint_tso":' 450140420784783361, '"checkpoint_time":' '"2024-05-31' '17:57:12.315",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375100620834952792, '"namespace":' '"default",' '"id":' '"939b98cd-a251-46b2-a26f-5b057f34e9bb",' '"state":' '"normal",' '"checkpoint_tso":' 450140420784783361, '"checkpoint_time":' '"2024-05-31' '17:57:12.315",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later [2024/05/31 17:57:23.191 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:57:23.193 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/31 17:57:23.357 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 17:57:23.419 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later [2024/05/31 17:57:23.551 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 17:57:23.552 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 17:57:23.630 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 17:57:23.632 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 17:57:23.755 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 17:57:23.831 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:23 GMT < Content-Length: 883 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c034b8e4-63b2-4445-80f0-f2830383ef91 {"id":"c034b8e4-63b2-4445-80f0-f2830383ef91","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318fce142391ca c034b8e4-63b2-4445-80f0-f2830383ef91 /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c034b8e4-63b2-4445-80f0-f2830383ef91 {"id":"c034b8e4-63b2-4445-80f0-f2830383ef91","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318fce142391ca c034b8e4-63b2-4445-80f0-f2830383ef91 /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c034b8e4-63b2-4445-80f0-f2830383ef91 {"id":"c034b8e4-63b2-4445-80f0-f2830383ef91","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318fce142391ca c034b8e4-63b2-4445-80f0-f2830383ef91 /tidb/cdc/default/default/upstream/7375100422790484190 {"id":7375100422790484190,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6729.out cli changefeed create --start-ts=0 '--sink-uri=kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' -c changefeed-initialize-error Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table common_1.finish_mark not exists for 3-th check, retry later [2024/05/31 17:57:23.973 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 17:57:23.974 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 17:57:24.066 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 17:57:24.068 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 17:57:24.165 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7375100422790484190,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:57:24.147134285+08:00","start_ts":450140423855276037,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140423855276037,"checkpoint_ts":450140423855276037,"checkpoint_time":"2024-05-31 17:57:24.028"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Debugger for raftstore-v2 is used [2024/05/31 17:57:24.262 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/31 17:57:24.357 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 17:57:24.358 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] table mark.finish_mark_3 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3854839c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn, pid:4077, start at 2024-05-31 17:57:22.29688827 +0800 CST m=+4.369244390 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:22.322 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:22.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:22.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3854839c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn, pid:4077, start at 2024-05-31 17:57:22.29688827 +0800 CST m=+4.369244390 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:22.322 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:22.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:22.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38548344000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9jk86-fnhxn, pid:4112, start at 2024-05-31 17:57:22.278798989 +0800 CST m=+4.256914688 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:22.302 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:22.308 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:22.308 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/31 17:57:24.476 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 17:57:24.478 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 17:57:24.551 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 17:57:24.687 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 17:57:24.751 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 17:57:24.752 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_1.usertable not exists for 2-th check, retry later + info='{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450140423855276037, "checkpoint_time": "2024-05-31 17:57:24.028", "error": { "time": "2024-05-31T17:57:24.32510275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450140423855276037, "checkpoint_time": "2024-05-31 17:57:24.028", "error": { "time": "2024-05-31T17:57:24.32510275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450140423855276037, "checkpoint_time": "2024-05-31 17:57:24.028", "error": { "time": "2024-05-31T17:57:24.32510275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 450140423855276037, '"checkpoint_time":' '"2024-05-31' '17:57:24.028",' '"error":' '{' '"time":' '"2024-05-31T17:57:24.32510275+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later check diff successfully table common_1.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table mark.finish_mark_3 not exists for 4-th check, retry later [Fri May 31 17:57:26 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.54505452.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later table common_1.finish_mark not exists for 5-th check, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450140423855276037, "checkpoint_time": "2024-05-31 17:57:24.028", "error": { "time": "2024-05-31T17:57:24.32510275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450140423855276037, "checkpoint_time": "2024-05-31 17:57:24.028", "error": { "time": "2024-05-31T17:57:24.32510275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 450140423855276037, "checkpoint_time": "2024-05-31 17:57:24.028", "error": { "time": "2024-05-31T17:57:24.32510275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 450140423855276037, '"checkpoint_time":' '"2024-05-31' '17:57:24.028",' '"error":' '{' '"time":' '"2024-05-31T17:57:24.32510275+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3854dba00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:16941, start at 2024-05-31 17:57:27.916131622 +0800 CST m=+1.825090883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:27.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:27.912 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:27.912 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3854dba00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:16941, start at 2024-05-31 17:57:27.916131622 +0800 CST m=+1.825090883 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:27.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:27.912 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:27.912 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3854db7c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:17000, start at 2024-05-31 17:57:27.941705789 +0800 CST m=+1.798688052 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:27.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:27.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:27.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:29 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99c1b82c-b1e3-4312-beb1-5a404986409a {"id":"99c1b82c-b1e3-4312-beb1-5a404986409a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149447} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14ffa797 99c1b82c-b1e3-4312-beb1-5a404986409a /tidb/cdc/default/default/upstream/7375100663103636413 {"id":7375100663103636413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99c1b82c-b1e3-4312-beb1-5a404986409a {"id":"99c1b82c-b1e3-4312-beb1-5a404986409a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149447} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14ffa797 99c1b82c-b1e3-4312-beb1-5a404986409a /tidb/cdc/default/default/upstream/7375100663103636413 {"id":7375100663103636413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/99c1b82c-b1e3-4312-beb1-5a404986409a {"id":"99c1b82c-b1e3-4312-beb1-5a404986409a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149447} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce14ffa797 99c1b82c-b1e3-4312-beb1-5a404986409a /tidb/cdc/default/default/upstream/7375100663103636413 {"id":7375100663103636413,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later Create changefeed successfully! ID: 9928c394-ff06-445d-bc68-cf45277d0d1f Info: {"upstream_id":7375100663103636413,"namespace":"default","id":"9928c394-ff06-445d-bc68-cf45277d0d1f","sink_uri":"kafka://127.0.0.1:9092/ticdc-force_replicate_table-test-26537?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:57:29.986363531+08:00","start_ts":450140424510898178,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140424510898178,"checkpoint_ts":450140424510898178,"checkpoint_time":"2024-05-31 17:57:26.529"} [Fri May 31 17:57:30 CST 2024] <<<<<< START kafka consumer in force_replicate_table case >>>>>> consumer replica config found: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml [Fri May 31 17:57:30 CST 2024] <<<<<< START cdc server in kafka_simple_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.1842318425.out server --log-file /tmp/tidb_cdc_test/kafka_simple_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_claim_check/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' table common_1.finish_mark exists check diff successfully + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... table mark.finish_mark_3 not exists for 6-th check, retry later wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later [2024/05/31 17:57:31.304 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] table force_replicate_table.t0 not exists for 1-th check, retry later table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later [2024/05/31 17:57:31.516 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:57:31.520 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:57:31.721 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:57:31.722 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:57:31.725 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:57:31.726 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:57:31 CST 2024] <<<<<< run test case common_1 success! >>>>>> [2024/05/31 17:57:31.925 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:57:31.945 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 17:57:32.140 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:57:32.141 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:57:32.166 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:57:32.167 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450140425847570437, "checkpoint_time": "2024-05-31 17:57:31.628", "error": null }' + echo '{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450140425847570437, "checkpoint_time": "2024-05-31 17:57:31.628", "error": null }' { "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450140425847570437, "checkpoint_time": "2024-05-31 17:57:31.628", "error": null } ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450140425847570437, '"checkpoint_time":' '"2024-05-31' '17:57:31.628",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450140425847570437, '"checkpoint_time":' '"2024-05-31' '17:57:31.628",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6882.out cli changefeed pause -c changefeed-initialize-error PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:57:32.526 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 17:57:32.650 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 17:57:32.861 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:57:32.862 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:57:32.899 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 17:57:32.900 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:57:33.050 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] check diff failed 4-th time, retry later [2024/05/31 17:57:33.087 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:57:33.239 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:57:33.240 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 17:57:33.280 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:57:33.281 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later [2024/05/31 17:57:33.426 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 17:57:33.477 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 17:57:33.629 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 17:57:33.630 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:57:33.701 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 17:57:33.702 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:33 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c67b1e9f-c22b-4f38-8466-335ff273d39b {"id":"c67b1e9f-c22b-4f38-8466-335ff273d39b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1519c783 c67b1e9f-c22b-4f38-8466-335ff273d39b /tidb/cdc/default/default/upstream/7375100691699608738 {"id":7375100691699608738,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c67b1e9f-c22b-4f38-8466-335ff273d39b {"id":"c67b1e9f-c22b-4f38-8466-335ff273d39b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1519c783 c67b1e9f-c22b-4f38-8466-335ff273d39b /tidb/cdc/default/default/upstream/7375100691699608738 {"id":7375100691699608738,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c67b1e9f-c22b-4f38-8466-335ff273d39b {"id":"c67b1e9f-c22b-4f38-8466-335ff273d39b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1519c783 c67b1e9f-c22b-4f38-8466-335ff273d39b /tidb/cdc/default/default/upstream/7375100691699608738 {"id":7375100691699608738,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.18468.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:57:33.849 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 17:57:33.934 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 17:57:34.065 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:57:34.066 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:57:34.164 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 17:57:34.166 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 17:57:34.274 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] + info='{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450140425847570437, "checkpoint_time": "2024-05-31 17:57:31.628", "error": { "time": "2024-05-31T17:57:24.32510275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450140425847570437, "checkpoint_time": "2024-05-31 17:57:31.628", "error": { "time": "2024-05-31T17:57:24.32510275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 450140425847570437, "checkpoint_time": "2024-05-31 17:57:31.628", "error": { "time": "2024-05-31T17:57:24.32510275+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 450140425847570437, '"checkpoint_time":' '"2024-05-31' '17:57:31.628",' '"error":' '{' '"time":' '"2024-05-31T17:57:24.32510275+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 450140425847570437, '"checkpoint_time":' '"2024-05-31' '17:57:31.628",' '"error":' '{' '"time":' '"2024-05-31T17:57:24.32510275+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6943.out cli changefeed resume -c changefeed-initialize-error [2024/05/31 17:57:34.396 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 17:57:34.502 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:57:34.503 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 8-th check, retry later [2024/05/31 17:57:34.640 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/31 17:57:34.641 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:57:34.730 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] PASS [2024/05/31 17:57:34.892 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 17:57:34.970 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 17:57:34.971 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] check diff failed 5-th time, retry later coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + tso='450140426420092932 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140426420092932 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.18505.out cli changefeed create --start-ts=450140426420092932 '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-30390?protocol=simple' -c kafka-simple-claim-check --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/conf/changefeed.toml [2024/05/31 17:57:35.128 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 17:57:35.129 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 17:57:35.190 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 17:57:35.322 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 17:57:35.375 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 17:57:35.376 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 17:57:35.514 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 17:57:35.515 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 17:57:35.563 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 17:57:35.720 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/31 17:57:35.764 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 17:57:35.765 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] Create changefeed successfully! ID: kafka-simple-claim-check Info: {"upstream_id":7375100691699608738,"namespace":"default","id":"kafka-simple-claim-check","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-30390?protocol=simple","create_time":"2024-05-31T17:57:35.703720503+08:00","start_ts":450140426420092932,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140426420092932,"checkpoint_ts":450140426420092932,"checkpoint_time":"2024-05-31 17:57:33.812"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later check diff successfully Verifying downstream PD is started... [2024/05/31 17:57:35.910 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 17:57:35.911 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 17:57:35.946 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] wait process cdc.test exit for 1-th time... [2024/05/31 17:57:36.107 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 17:57:36.138 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 17:57:36.139 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 17:57:36.301 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/31 17:57:36.302 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450140425847570437, "checkpoint_time": "2024-05-31 17:57:31.628", "error": null }' + echo '{ "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450140425847570437, "checkpoint_time": "2024-05-31 17:57:31.628", "error": null }' { "upstream_id": 7375100422790484190, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 450140425847570437, "checkpoint_time": "2024-05-31 17:57:31.628", "error": null } ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450140425847570437, '"checkpoint_time":' '"2024-05-31' '17:57:31.628",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375100422790484190, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 450140425847570437, '"checkpoint_time":' '"2024-05-31' '17:57:31.628",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6999.out cli changefeed remove -c changefeed-initialize-error wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 9-th check, retry later + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:57:37 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> check diff successfully wait process cdc.test exit for 1-th time... Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 450140425847570437 SinkURI: kafka://127.0.0.1:9092/ticdc-sink-retry-test-11799?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:57:38 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: kafka... <<================= ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table mark.finish_mark_3 not exists for 10-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:57:40 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 11-th check, retry later % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 1415 100 678 100 737 1161 1262 --:--:-- --:--:-- --:--:-- 1261 100 1415 100 678 100 737 1161 1262 --:--:-- --:--:-- --:--:-- 1261 HTTP/1.1 201 Created Date: Fri, 31 May 2024 09:57:40 GMT Location: http://localhost:8083/connectors/my-connector Content-Type: application/json Content-Length: 678 Server: Jetty(9.4.51.v20230217) {"name":"my-connector","config":{"connector.class":"io.debezium.connector.mysql.MySqlConnector","tasks.max":"1","database.hostname":"127.0.0.1","database.port":"3310","database.user":"debezium","database.password":"dbz","database.server.id":"184054","topic.prefix":"dbserver1","schema.history.internal.kafka.bootstrap.servers":"127.0.0.1:9092","schema.history.internal.kafka.topic":"schemahistory.test","transforms":"x","transforms.x.type":"org.apache.kafka.connect.transforms.RegexRouter","transforms.x.regex":"(.*)","transforms.x.replacement":"output_debezium","binary.handling.mode":"base64","decimal.handling.mode":"double","name":"my-connector"},"tasks":[],"type":"source"}The 1 times to try to start tidb cluster... table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 12-th check, retry later [2024/05/31 17:57:42.297 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/05/31 17:57:42.525 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:57:42.525 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:57:42.715 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:57:42.717 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:57:42.727 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:57:42.729 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:57:42.914 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:57:42.947 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 17:57:43.097 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:57:43.098 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:57:43.140 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:57:43.143 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:57:43.289 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 17:57:43.337 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 17:57:43.485 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:57:43.488 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:57:43.561 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:57:43.564 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3855c98c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:23569, start at 2024-05-31 17:57:43.16252911 +0800 CST m=+1.908305550 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:43.169 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:43.139 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:43.139 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3855c98c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:23569, start at 2024-05-31 17:57:43.16252911 +0800 CST m=+1.908305550 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:43.169 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:43.139 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:43.139 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3855c9f40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:23639, start at 2024-05-31 17:57:43.194772616 +0800 CST m=+1.877423844 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:43.203 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:43.165 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:43.165 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.18560.out cli changefeed pause -c kafka-simple-claim-check [2024/05/31 17:57:44.036 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 17:57:44.124 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] table force_replicate_table.t6 not exists for 2-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/31 17:57:44.322 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 17:57:44.325 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:57:44.435 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:57:44.437 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:57:44.641 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 13-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/31 17:57:44.774 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 17:57:45.035 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:57:45.039 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 17:57:45.123 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 17:57:45.126 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 17:57:45.447 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.24957.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:57:45.508 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.18597.out cli changefeed update -c kafka-simple-claim-check '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-30390?protocol=simple&max-message-bytes=2048' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/conf/changefeed.toml --no-confirm [2024/05/31 17:57:45.748 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 17:57:45.750 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 17:57:45.786 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 17:57:45.790 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:57:45.963 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/kafka-simple-claim-check-30390?protocol=simple To:kafka://127.0.0.1:9092/kafka-simple-claim-check-30390?protocol=simple&max-message-bytes=2048} {Type:update Path:[Config SyncPointInterval] From: To:0xc0036237f8} {Type:update Path:[Config SyncPointRetention] From: To:0xc003623808} {Type:update Path:[Config Consistent] From: To:0xc000a6c150} table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 [2024/05/31 17:57:46.086 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 17:57:46.172 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:57:46.175 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] Update changefeed config successfully! ID: kafka-simple-claim-check Info: {"upstream_id":7375100691699608738,"namespace":"default","id":"kafka-simple-claim-check","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-30390?protocol=simple\u0026max-message-bytes=2048","create_time":"2024-05-31T17:57:35.703720503+08:00","start_ts":450140426420092932,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":0,"checkpoint_ts":450140428858032135,"checkpoint_time":"2024-05-31 17:57:43.112"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [2024/05/31 17:57:46.390 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 17:57:46.392 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:57:46.394 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 17:57:46.712 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 17:57:46.716 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:57:43.608 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 13"] [2024/05/31 17:57:46.804 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:57:47.157 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] table mark.finish_mark_3 not exists for 14-th check, retry later [2024/05/31 17:57:47.288 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 17:57:47.293 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] + set +x + tso='450140429586006017 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140429586006017 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check.cli.18628.out cli changefeed resume -c kafka-simple-claim-check PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "mysql.db"="move_table" "dotransactions"="false" "recordcount"="10000" "scanproportion"="0" "threadcount"="10" "updateproportion"="0" "mysql.port"="4000" "operationcount"="0" "readallfields"="true" "insertproportion"="0" "mysql.user"="root" "workload"="core" "readproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" ********************************************** run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 [2024/05/31 17:57:48.609 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 34 another is 27"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38560d180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:9778, start at 2024-05-31 17:57:47.490694706 +0800 CST m=+24.115744637 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:47.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:47.462 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:47.462 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38560d180013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:9778, start at 2024-05-31 17:57:47.490694706 +0800 CST m=+24.115744637 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:47.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:47.462 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:47.462 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3854e7c40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-tr2x6-dnsh1, pid:9850, start at 2024-05-31 17:57:28.735453837 +0800 CST m=+5.301969982 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:28.744 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:28.740 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:28.740 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log table mark.finish_mark_3 not exists for 15-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38561ef40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k, pid:21682, start at 2024-05-31 17:57:48.614752689 +0800 CST m=+2.002501998 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:48.627 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:48.605 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:48.605 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38561ef40009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k, pid:21682, start at 2024-05-31 17:57:48.614752689 +0800 CST m=+2.002501998 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:48.627 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:48.605 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:48.605 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38561ee40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k, pid:21753, start at 2024-05-31 17:57:48.649139041 +0800 CST m=+1.983758109 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:48.656 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:48.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:48.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } Run finished, takes 1.337403653s INSERT - Takes(s): 1.3, Count: 10000, OPS: 7507.6, Avg(us): 1293, Min(us): 828, Max(us): 7804, 95th(us): 2000, 99th(us): 3000 [Fri May 31 17:57:49 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2504125043.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] // dir run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 [Pipeline] } [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) table test.finish_mark not exists for 1-th check, retry later [Pipeline] // withCredentials [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // container [Pipeline] } [Pipeline] } [Fri May 31 17:57:50 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.2353923541.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table mark.finish_mark_3 not exists for 16-th check, retry later [Pipeline] // podTemplate [Pipeline] // withEnv + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.23090.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [Pipeline] } run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 start tidb cluster in /tmp/tidb_cdc_test/debezium Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [Pipeline] // withEnv [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.finish_mark not exists for 2-th check, retry later run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 table mark.finish_mark_3 not exists for 17-th check, retry later + set +x + tso='450140431045099521 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140431045099521 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:57:52 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.2313323135.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:52 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dff5773b-f284-4c55-bdc3-56806bd0484a {"id":"dff5773b-f284-4c55-bdc3-56806bd0484a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd8b8 dff5773b-f284-4c55-bdc3-56806bd0484a /tidb/cdc/default/default/upstream/7375100747602836250 {"id":7375100747602836250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dff5773b-f284-4c55-bdc3-56806bd0484a {"id":"dff5773b-f284-4c55-bdc3-56806bd0484a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd8b8 dff5773b-f284-4c55-bdc3-56806bd0484a /tidb/cdc/default/default/upstream/7375100747602836250 {"id":7375100747602836250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dff5773b-f284-4c55-bdc3-56806bd0484a {"id":"dff5773b-f284-4c55-bdc3-56806bd0484a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd8b8 dff5773b-f284-4c55-bdc3-56806bd0484a /tidb/cdc/default/default/upstream/7375100747602836250 {"id":7375100747602836250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.25091.out cli changefeed create --start-ts=450140429586006017 '--sink-uri=kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: be8d4912-6a98-4a11-b82c-aa0443f122b0 Info: {"upstream_id":7375100747602836250,"namespace":"default","id":"be8d4912-6a98-4a11-b82c-aa0443f122b0","sink_uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:57:53.258484839+08:00","start_ts":450140429586006017,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140429586006017,"checkpoint_ts":450140429586006017,"checkpoint_time":"2024-05-31 17:57:45.889"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:54 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cecbef4-2940-4c5e-a6c0-0eb53d981818 {"id":"6cecbef4-2940-4c5e-a6c0-0eb53d981818","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149471} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce150d17c9 6cecbef4-2940-4c5e-a6c0-0eb53d981818 /tidb/cdc/default/default/upstream/7375100678750199330 {"id":7375100678750199330,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cecbef4-2940-4c5e-a6c0-0eb53d981818 {"id":"6cecbef4-2940-4c5e-a6c0-0eb53d981818","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149471} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce150d17c9 6cecbef4-2940-4c5e-a6c0-0eb53d981818 /tidb/cdc/default/default/upstream/7375100678750199330 {"id":7375100678750199330,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6cecbef4-2940-4c5e-a6c0-0eb53d981818 {"id":"6cecbef4-2940-4c5e-a6c0-0eb53d981818","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149471} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce150d17c9 6cecbef4-2940-4c5e-a6c0-0eb53d981818 /tidb/cdc/default/default/upstream/7375100678750199330 {"id":7375100678750199330,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.23811.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-rocks-test-9033?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table test.finish_mark exists check diff successfully Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... Create changefeed successfully! ID: a31b8ee6-32a9-41af-bba7-df2d7d5575a6 Info: {"upstream_id":7375100678750199330,"namespace":"default","id":"a31b8ee6-32a9-41af-bba7-df2d7d5575a6","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-rocks-test-9033?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:57:54.522868042+08:00","start_ts":450140431807152133,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140431807152133,"checkpoint_ts":450140431807152133,"checkpoint_time":"2024-05-31 17:57:54.362"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 18-th check, retry later + set +x [Fri May 31 17:57:54 CST 2024] <<<<<< START kafka consumer in move_table case >>>>>> [Fri May 31 17:57:54 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2512925131.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... [2024/05/31 17:57:54.761 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m7.255575326s"] table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark_5 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:57:55 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> run task successfully + set +x [Fri May 31 17:57:55 CST 2024] <<<<<< START kafka consumer in multi_rocks case >>>>>> ***************** properties ***************** "mysql.db"="multi_rocks" "scanproportion"="0" "readallfields"="true" "readproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "operationcount"="0" "insertproportion"="0" "updateproportion"="0" "table"="a1" "workload"="core" "dotransactions"="false" "threadcount"="2" "mysql.port"="4000" "recordcount"="1000" "mysql.user"="root" ********************************************** wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:55 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27c37a2f-1a09-457c-a31f-b2d71dc54e17 {"id":"27c37a2f-1a09-457c-a31f-b2d71dc54e17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149473} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1568169c 27c37a2f-1a09-457c-a31f-b2d71dc54e17 /tidb/cdc/default/default/upstream/7375100789552673971 {"id":7375100789552673971,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27c37a2f-1a09-457c-a31f-b2d71dc54e17 {"id":"27c37a2f-1a09-457c-a31f-b2d71dc54e17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149473} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1568169c 27c37a2f-1a09-457c-a31f-b2d71dc54e17 /tidb/cdc/default/default/upstream/7375100789552673971 {"id":7375100789552673971,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/27c37a2f-1a09-457c-a31f-b2d71dc54e17 {"id":"27c37a2f-1a09-457c-a31f-b2d71dc54e17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149473} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1568169c 27c37a2f-1a09-457c-a31f-b2d71dc54e17 /tidb/cdc/default/default/upstream/7375100789552673971 {"id":7375100789552673971,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run finished, takes 503.992266ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 2034.5, Avg(us): 978, Min(us): 738, Max(us): 12350, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "recordcount"="1000" "updateproportion"="0" "table"="a2" "readproportion"="0" "mysql.user"="root" "scanproportion"="0" "dotransactions"="false" "insertproportion"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "mysql.db"="multi_rocks" "readallfields"="true" "requestdistribution"="uniform" "operationcount"="0" "mysql.port"="4000" "workload"="core" ********************************************** table mark.finish_mark_3 not exists for 19-th check, retry later wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Run finished, takes 529.771592ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1961.0, Avg(us): 1024, Min(us): 734, Max(us): 19709, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "operationcount"="0" "workload"="core" "threadcount"="2" "insertproportion"="0" "readproportion"="0" "scanproportion"="0" "table"="a3" "updateproportion"="0" "mysql.db"="multi_rocks" "recordcount"="1000" "mysql.port"="4000" "readallfields"="true" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "mysql.user"="root" "dotransactions"="false" ********************************************** [Fri May 31 17:57:57 CST 2024] <<<<<< START kafka consumer in capture_session_done_during_task case >>>>>> lease 22318fce1568169c revoked table mark.finish_mark_5 exists table mark.finish_mark not exists for 1-th check, retry later Run finished, takes 556.297951ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1850.6, Avg(us): 1081, Min(us): 772, Max(us): 15828, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "recordcount"="1000" "mysql.port"="4000" "threadcount"="2" "insertproportion"="0" "dotransactions"="false" "readproportion"="0" "scanproportion"="0" "mysql.user"="root" "workload"="core" "requestdistribution"="uniform" "operationcount"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "mysql.db"="multi_rocks" "table"="a4" "updateproportion"="0" ********************************************** + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:57:57 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/be8d4912-6a98-4a11-b82c-aa0443f122b0 {UpstreamID:7375100747602836250 Namespace:default ID:be8d4912-6a98-4a11-b82c-aa0443f122b0 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:57:53.258484839 +0800 CST StartTs:450140429586006017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037c0a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140431512764422} {CheckpointTs:450140430254473347 MinTableBarrierTs:450140432574447626 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4d71ac0c-535e-48cc-9fb8-7ef552647faa {"id":"4d71ac0c-535e-48cc-9fb8-7ef552647faa","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149475} /tidb/cdc/default/__cdc_meta__/capture/dff5773b-f284-4c55-bdc3-56806bd0484a {"id":"dff5773b-f284-4c55-bdc3-56806bd0484a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd8b8 dff5773b-f284-4c55-bdc3-56806bd0484a /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd908 4d71ac0c-535e-48cc-9fb8-7ef552647faa /tidb/cdc/default/default/changefeed/info/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"upstream-id":7375100747602836250,"namespace":"default","changefeed-id":"be8d4912-6a98-4a11-b82c-aa0443f122b0","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:57:53.258484839+08:00","start-ts":450140429586006017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140431512764422} /tidb/cdc/default/default/changefeed/status/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":450140430254473347,"min-table-barrier-ts":450140432574447626,"admin-job-type":0} /tidb/cdc/default/default/task/position/4d71ac0c-535e-48cc-9fb8-7ef552647faa/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dff5773b-f284-4c55-bdc3-56806bd0484a/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100747602836250 {"id":7375100747602836250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/be8d4912-6a98-4a11-b82c-aa0443f122b0 {UpstreamID:7375100747602836250 Namespace:default ID:be8d4912-6a98-4a11-b82c-aa0443f122b0 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:57:53.258484839 +0800 CST StartTs:450140429586006017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037c0a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140431512764422} {CheckpointTs:450140430254473347 MinTableBarrierTs:450140432574447626 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4d71ac0c-535e-48cc-9fb8-7ef552647faa {"id":"4d71ac0c-535e-48cc-9fb8-7ef552647faa","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149475} /tidb/cdc/default/__cdc_meta__/capture/dff5773b-f284-4c55-bdc3-56806bd0484a {"id":"dff5773b-f284-4c55-bdc3-56806bd0484a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd8b8 dff5773b-f284-4c55-bdc3-56806bd0484a /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd908 4d71ac0c-535e-48cc-9fb8-7ef552647faa /tidb/cdc/default/default/changefeed/info/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"upstream-id":7375100747602836250,"namespace":"default","changefeed-id":"be8d4912-6a98-4a11-b82c-aa0443f122b0","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:57:53.258484839+08:00","start-ts":450140429586006017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140431512764422} /tidb/cdc/default/default/changefeed/status/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":450140430254473347,"min-table-barrier-ts":450140432574447626,"admin-job-type":0} /tidb/cdc/default/default/task/position/4d71ac0c-535e-48cc-9fb8-7ef552647faa/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dff5773b-f284-4c55-bdc3-56806bd0484a/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100747602836250 {"id":7375100747602836250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/be8d4912-6a98-4a11-b82c-aa0443f122b0 {UpstreamID:7375100747602836250 Namespace:default ID:be8d4912-6a98-4a11-b82c-aa0443f122b0 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:57:53.258484839 +0800 CST StartTs:450140429586006017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037c0a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140431512764422} {CheckpointTs:450140430254473347 MinTableBarrierTs:450140432574447626 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4d71ac0c-535e-48cc-9fb8-7ef552647faa {"id":"4d71ac0c-535e-48cc-9fb8-7ef552647faa","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149475} /tidb/cdc/default/__cdc_meta__/capture/dff5773b-f284-4c55-bdc3-56806bd0484a {"id":"dff5773b-f284-4c55-bdc3-56806bd0484a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd8b8 dff5773b-f284-4c55-bdc3-56806bd0484a /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd908 4d71ac0c-535e-48cc-9fb8-7ef552647faa /tidb/cdc/default/default/changefeed/info/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"upstream-id":7375100747602836250,"namespace":"default","changefeed-id":"be8d4912-6a98-4a11-b82c-aa0443f122b0","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:57:53.258484839+08:00","start-ts":450140429586006017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140431512764422} /tidb/cdc/default/default/changefeed/status/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":450140430254473347,"min-table-barrier-ts":450140432574447626,"admin-job-type":0} /tidb/cdc/default/default/task/position/4d71ac0c-535e-48cc-9fb8-7ef552647faa/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dff5773b-f284-4c55-bdc3-56806bd0484a/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100747602836250 {"id":7375100747602836250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Fri May 31 17:57:57 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2524425246.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Run finished, takes 538.845038ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1913.9, Avg(us): 1042, Min(us): 732, Max(us): 16268, 95th(us): 2000, 99th(us): 2000 table capture_session_done_during_task.t exists check diff failed 1-th time, retry later ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.user"="root" "table"="a5" "requestdistribution"="uniform" "recordcount"="1000" "readproportion"="0" "workload"="core" "mysql.port"="4000" "operationcount"="0" "threadcount"="2" "scanproportion"="0" "dotransactions"="false" "readallfields"="true" "insertproportion"="0" "mysql.db"="multi_rocks" "updateproportion"="0" ********************************************** Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Run finished, takes 529.776812ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1952.6, Avg(us): 1023, Min(us): 747, Max(us): 17551, 95th(us): 2000, 99th(us): 2000 table multi_rocks.finish_mark not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 20-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri May 31 17:57:59 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> table mark.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3856bb94000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:33203, start at 2024-05-31 17:57:58.640682419 +0800 CST m=+2.049018763 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:58.647 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:58.629 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:58.629 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3856bb94000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:33203, start at 2024-05-31 17:57:58.640682419 +0800 CST m=+2.049018763 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:58.647 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:58.629 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:58.629 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3856bdfc000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-c379v-2wkq6, pid:33257, start at 2024-05-31 17:57:58.795051369 +0800 CST m=+2.150103738 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-17:59:58.806 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:57:58.783 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:47:58.783 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/debezium/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/debezium/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/debezium/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/debezium/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/debezium/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later table mark.finish_mark_3 not exists for 21-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:58:00 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/be8d4912-6a98-4a11-b82c-aa0443f122b0 {UpstreamID:7375100747602836250 Namespace:default ID:be8d4912-6a98-4a11-b82c-aa0443f122b0 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:57:53.258484839 +0800 CST StartTs:450140429586006017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e30000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140431512764422} {CheckpointTs:450140433361141772 MinTableBarrierTs:450140433361141772 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/033559f0-a361-4081-9b59-89d41016cea8 {"id":"033559f0-a361-4081-9b59-89d41016cea8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149478} /tidb/cdc/default/__cdc_meta__/capture/4d71ac0c-535e-48cc-9fb8-7ef552647faa {"id":"4d71ac0c-535e-48cc-9fb8-7ef552647faa","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149475} /tidb/cdc/default/__cdc_meta__/capture/dff5773b-f284-4c55-bdc3-56806bd0484a {"id":"dff5773b-f284-4c55-bdc3-56806bd0484a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd8b8 dff5773b-f284-4c55-bdc3-56806bd0484a /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd908 4d71ac0c-535e-48cc-9fb8-7ef552647faa /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd933 033559f0-a361-4081-9b59-89d41016cea8 /tidb/cdc/default/default/changefeed/info/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"upstream-id":7375100747602836250,"namespace":"default","changefeed-id":"be8d4912-6a98-4a11-b82c-aa0443f122b0","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:57:53.258484839+08:00","start-ts":450140429586006017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140431512764422} /tidb/cdc/default/default/changefeed/status/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":450140433361141772,"min-table-barrier-ts":450140433361141772,"admin-job-type":0} /tidb/cdc/default/default/task/position/033559f0-a361-4081-9b59-89d41016cea8/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4d71ac0c-535e-48cc-9fb8-7ef552647faa/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dff5773b-f284-4c55-bdc3-56806bd0484a/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100747602836250 {"id":7375100747602836250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/be8d4912-6a98-4a11-b82c-aa0443f122b0 {UpstreamID:7375100747602836250 Namespace:default ID:be8d4912-6a98-4a11-b82c-aa0443f122b0 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:57:53.258484839 +0800 CST StartTs:450140429586006017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e30000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140431512764422} {CheckpointTs:450140433361141772 MinTableBarrierTs:450140433361141772 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/033559f0-a361-4081-9b59-89d41016cea8 {"id":"033559f0-a361-4081-9b59-89d41016cea8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149478} /tidb/cdc/default/__cdc_meta__/capture/4d71ac0c-535e-48cc-9fb8-7ef552647faa {"id":"4d71ac0c-535e-48cc-9fb8-7ef552647faa","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149475} /tidb/cdc/default/__cdc_meta__/capture/dff5773b-f284-4c55-bdc3-56806bd0484a {"id":"dff5773b-f284-4c55-bdc3-56806bd0484a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd8b8 dff5773b-f284-4c55-bdc3-56806bd0484a /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd908 4d71ac0c-535e-48cc-9fb8-7ef552647faa /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd933 033559f0-a361-4081-9b59-89d41016cea8 /tidb/cdc/default/default/changefeed/info/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"upstream-id":7375100747602836250,"namespace":"default","changefeed-id":"be8d4912-6a98-4a11-b82c-aa0443f122b0","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:57:53.258484839+08:00","start-ts":450140429586006017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-thresh+ grep -q 'failed to get info:' old":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140431512764422} /tidb/cdc/default/default/changefeed/status/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":450140433361141772,"min-table-barrier-ts":450140433361141772,"admin-job-type":0} /tidb/cdc/default/default/task/position/033559f0-a361-4081-9b59-89d41016cea8/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4d71ac0c-535e-48cc-9fb8-7ef552647faa/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dff5773b-f284-4c55-bdc3-56806bd0484a/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100747602836250 {"id":7375100747602836250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/be8d4912-6a98-4a11-b82c-aa0443f122b0 {UpstreamID:7375100747602836250 Namespace:default ID:be8d4912-6a98-4a11-b82c-aa0443f122b0 SinkURI:kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 CreateTime:2024-05-31 17:57:53.258484839 +0800 CST StartTs:450140429586006017 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e30000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140431512764422} {CheckpointTs:450140433361141772 MinTableBarrierTs:450140433361141772 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/033559f0-a361-4081-9b59-89d41016cea8 {"id":"033559f0-a361-4081-9b59-89d41016cea8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149478} /tidb/cdc/default/__cdc_meta__/capture/4d71ac0c-535e-48cc-9fb8-7ef552647faa {"id":"4d71ac0c-535e-48cc-9fb8-7ef552647faa","address":"127.0.0.1:8301","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149475} /tidb/cdc/default/__cdc_meta__/capture/dff5773b-f284-4c55-bdc3-56806bd0484a {"id":"dff5773b-f284-4c55-bdc3-56806bd0484a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149470} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd8b8 dff5773b-f284-4c55-bdc3-56806bd0484a /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd908 4d71ac0c-535e-48cc-9fb8-7ef552647faa /tidb/cdc/default/__cdc_meta__/owner/22318fce154dd933 033559f0-a361-4081-9b59-89d41016cea8 /tidb/cdc/default/default/changefeed/info/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"upstream-id":7375100747602836250,"namespace":"default","changefeed-id":"be8d4912-6a98-4a11-b82c-aa0443f122b0","sink-uri":"kafka://127.0.0.1:9092/ticdc-move-table-test-29694?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create-time":"2024-05-31T17:57:53.258484839+08:00","start-ts":450140429586006017,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-thresh+ grep -q 'etcd info' old":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140431512764422} /tidb/cdc/default/default/changefeed/status/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":450140433361141772,"min-table-barrier-ts":450140433361141772,"admin-job-type":0} /tidb/cdc/default/default/task/position/033559f0-a361-4081-9b59-89d41016cea8/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/4d71ac0c-535e-48cc-9fb8-7ef552647faa/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/dff5773b-f284-4c55-bdc3-56806bd0484a/be8d4912-6a98-4a11-b82c-aa0443f122b0 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375100747602836250 {"id":7375100747602836250,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table move_table.usertable exists go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/IBM/sarama v1.41.2 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240507075657-2bd4573edde2 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/99designs/keyring v1.2.1 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/go-playground/locales v0.14.1 table multi_rocks.finish_mark not exists for 2-th check, retry later go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [Fri May 31 17:58:02 CST 2024] <<<<<< START cdc server in debezium case >>>>>> check diff successfully check diff failed 1-th time, retry later table mark.finish_mark_3 not exists for 22-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3856d79c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:10853, start at 2024-05-31 17:58:00.461334113 +0800 CST m=+1.953575334 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:00.468 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:00.473 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:00.473 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3856d79c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:10853, start at 2024-05-31 17:58:00.461334113 +0800 CST m=+1.953575334 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:00.468 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:00.473 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:00.473 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3856d83c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:10934, start at 2024-05-31 17:58:00.504634914 +0800 CST m=+1.934423749 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:00.511 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:00.512 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:00.512 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.debezium.3463834640.out server --log-file /tmp/tidb_cdc_test/debezium/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/debezium/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/ardielle/ardielle-go v1.5.2 table multi_rocks.finish_mark not exists for 3-th check, retry later table mark.finish_mark_3 not exists for 23-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12263.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully wait process cdc.test exit for 1-th time... table multi_rocks.finish_mark not exists for 4-th check, retry later wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:58:06 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e39e40b0-ec7f-4606-b7c5-45aabfd93362 {"id":"e39e40b0-ec7f-4606-b7c5-45aabfd93362","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce158ed38a e39e40b0-ec7f-4606-b7c5-45aabfd93362 /tidb/cdc/default/default/upstream/7375100828178380564 {"id":7375100828178380564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e39e40b0-ec7f-4606-b7c5-45aabfd93362 {"id":"e39e40b0-ec7f-4606-b7c5-45aabfd93362","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce158ed38a e39e40b0-ec7f-4606-b7c5-45aabfd93362 /tidb/cdc/default/default/upstream/7375100828178380564 {"id":7375100828178380564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e39e40b0-ec7f-4606-b7c5-45aabfd93362 {"id":"e39e40b0-ec7f-4606-b7c5-45aabfd93362","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce158ed38a e39e40b0-ec7f-4606-b7c5-45aabfd93362 /tidb/cdc/default/default/upstream/7375100828178380564 {"id":7375100828178380564,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.debezium.cli.34684.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/output_ticdc?protocol=debezium&kafka-version=2.4.0' cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:58:06 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> Create changefeed successfully! ID: fcbb4f71-8880-4923-af4d-95d8bc8d2ddf Info: {"upstream_id":7375100828178380564,"namespace":"default","id":"fcbb4f71-8880-4923-af4d-95d8bc8d2ddf","sink_uri":"kafka://127.0.0.1:9092/output_ticdc?protocol=debezium\u0026kafka-version=2.4.0","create_time":"2024-05-31T17:58:06.48948016+08:00","start_ts":450140434944229380,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"debezium","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140434944229380,"checkpoint_ts":450140434944229380,"checkpoint_time":"2024-05-31 17:58:06.329"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 24-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450140434641453057 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 450140434641453057 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table multi_rocks.finish_mark not exists for 5-th check, retry later [Fri May 31 17:58:07 CST 2024] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1230512307.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20231117065153-a4f85c356873 go: downloading github.com/alecthomas/chroma v0.10.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/google/go-cmp v0.6.0 go: downloading github.com/segmentio/kafka-go v0.4.45 go: downloading github.com/google/uuid v1.3.1 go: downloading github.com/thessem/zap-prettyconsole v0.3.0 go: downloading go.uber.org/zap v1.26.0 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20231116213047-1f7c1e02bcd4 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading golang.org/x/sys v0.14.0 go: downloading github.com/Code-Hex/dd v1.1.0 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh: line 1: 18651 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... go: downloading github.com/klauspost/compress v1.17.1 go: downloading github.com/pierrec/lz4/v4 v4.1.15 go: downloading github.com/dlclark/regexp2 v1.4.0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_rocks.finish_mark exists table mark.finish_mark_3 not exists for 25-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... go: downloading github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 go: downloading golang.org/x/text v0.14.0 go: downloading golang.org/x/exp v0.0.0-20231006140011-7918f672742d go: downloading github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 go: downloading github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 go: downloading github.com/shirou/gopsutil/v3 v3.23.10 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20231114060955-8fc8a528217e go: downloading github.com/prometheus/client_golang v1.17.0 go: downloading github.com/golang/protobuf v1.5.3 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 go: downloading github.com/cockroachdb/errors v1.8.1 go: downloading github.com/prometheus/client_model v0.5.0 go: downloading google.golang.org/protobuf v1.31.0 go: downloading github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071 go: downloading google.golang.org/grpc v1.59.0 go: downloading github.com/cespare/xxhash/v2 v2.2.0 go: downloading github.com/prometheus/procfs v0.12.0 go: downloading github.com/prometheus/common v0.45.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f go: downloading github.com/cockroachdb/redact v1.0.8 go: downloading github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 go: downloading github.com/rogpeppe/go-internal v1.11.0 go: downloading github.com/tikv/pd/client v0.0.0-20231114041114-86831ce71865 go: downloading go.etcd.io/etcd/api/v3 v3.5.10 go: downloading go.etcd.io/etcd/client/v3 v3.5.10 go: downloading golang.org/x/sync v0.4.0 go: downloading github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.10 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b go: downloading google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b go: downloading golang.org/x/net v0.18.0 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:58:10 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:58:10 GMT < Content-Length: 859 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c87c7c5-0d16-4c2c-bebc-32ad0a42d30e {"id":"5c87c7c5-0d16-4c2c-bebc-32ad0a42d30e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149488} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce159c4897 5c87c7c5-0d16-4c2c-bebc-32ad0a42d30e /tidb/cdc/default/default/upstream/7375100841339931867 {"id":7375100841339931867,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c87c7c5-0d16-4c2c-bebc-32ad0a42d30e {"id":"5c87c7c5-0d16-4c2c-bebc-32ad0a42d30e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149488} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce159c4897 5c87c7c5-0d16-4c2c-bebc-32ad0a42d30e /tidb/cdc/default/default/upstream/7375100841339931867 {"id":7375100841339931867,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c87c7c5-0d16-4c2c-bebc-32ad0a42d30e {"id":"5c87c7c5-0d16-4c2c-bebc-32ad0a42d30e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149488} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce159c4897 5c87c7c5-0d16-4c2c-bebc-32ad0a42d30e /tidb/cdc/default/default/upstream/7375100841339931867 {"id":7375100841339931867,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12351.out cli changefeed create --start-ts=450140434641453057 '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-6413?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name table mark.finish_mark_3 not exists for 26-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385758bc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:8138, start at 2024-05-31 17:58:08.731891211 +0800 CST m=+2.075277444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:08.747 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:08.738 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:08.738 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385758bc0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:8138, start at 2024-05-31 17:58:08.731891211 +0800 CST m=+2.075277444 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:08.747 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:08.738 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:08.738 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38575a54000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:8209, start at 2024-05-31 17:58:08.799554093 +0800 CST m=+2.086997525 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:08.806 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:08.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:08.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/kafka_simple_claim_check_avro Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7375100841339931867,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-6413?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:58:11.262524872+08:00","start_ts":450140434641453057,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140434641453057,"checkpoint_ts":450140434641453057,"checkpoint_time":"2024-05-31 17:58:05.174"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... The 1 times to try to start tidb cluster... + set +x [Fri May 31 17:58:12 CST 2024] <<<<<< START kafka consumer in cli_with_auth case >>>>>> table test.simple not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 27-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9538.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/multi_source Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x + tso='450140436781334529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140436781334529 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:58:14 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.95739575.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 28-th check, retry later table test.simple not exists for 2-th check, retry later [2024/05/31 17:58:14.521 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/31 17:58:14.523 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/31 17:58:14.523 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=dff5773b-f284-4c55-bdc3-56806bd0484a] [2024/05/31 17:58:14.524 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/31 17:58:14.524 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/31 17:58:14.647 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=be8d4912-6a98-4a11-b82c-aa0443f122b0] [captureID=033559f0-a361-4081-9b59-89d41016cea8] [processorDetail="{\"table_ids\":[]}"] [2024/05/31 17:58:14.796 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=be8d4912-6a98-4a11-b82c-aa0443f122b0] [captureID=4d71ac0c-535e-48cc-9fb8-7ef552647faa] [processorDetail="{\"table_ids\":[108]}"] [2024/05/31 17:58:14.947 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=be8d4912-6a98-4a11-b82c-aa0443f122b0] [captureID=dff5773b-f284-4c55-bdc3-56806bd0484a] [processorDetail="{\"table_ids\":[106]}"] [2024/05/31 17:58:14.947 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"033559f0-a361-4081-9b59-89d41016cea8\":[],\"4d71ac0c-535e-48cc-9fb8-7ef552647faa\":[{\"ID\":108,\"Changefeed\":\"be8d4912-6a98-4a11-b82c-aa0443f122b0\"}],\"dff5773b-f284-4c55-bdc3-56806bd0484a\":[{\"ID\":106,\"Changefeed\":\"be8d4912-6a98-4a11-b82c-aa0443f122b0\"}]}"] [2024/05/31 17:58:14.947 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=be8d4912-6a98-4a11-b82c-aa0443f122b0&target-cp-id=dff5773b-f284-4c55-bdc3-56806bd0484a&table-id=108"] [2024/05/31 17:58:14.996 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/31 17:58:14.996 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=4d71ac0c-535e-48cc-9fb8-7ef552647faa] [targetCapture=dff5773b-f284-4c55-bdc3-56806bd0484a] table move_table.check1 exists check diff successfully Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 29-th check, retry later table test.simple exists table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140435165741083, "checkpoint_time": "2024-05-31 17:58:07.174", "error": null }' + echo '{ "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140435165741083, "checkpoint_time": "2024-05-31 17:58:07.174", "error": null }' { "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140435165741083, "checkpoint_time": "2024-05-31 17:58:07.174", "error": null } ++ echo '{' '"upstream_id":' 7375100841339931867, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140435165741083, '"checkpoint_time":' '"2024-05-31' '17:58:07.174",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375100841339931867, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140435165741083, '"checkpoint_time":' '"2024-05-31' '17:58:07.174",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:58:17 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03843d81-06ff-42d9-aeaf-16739fa42d8d {"id":"03843d81-06ff-42d9-aeaf-16739fa42d8d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149495} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce15ba3997 03843d81-06ff-42d9-aeaf-16739fa42d8d /tidb/cdc/default/default/upstream/7375100874550838532 {"id":7375100874550838532,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03843d81-06ff-42d9-aeaf-16739fa42d8d {"id":"03843d81-06ff-42d9-aeaf-16739fa42d8d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149495} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce15ba3997 03843d81-06ff-42d9-aeaf-16739fa42d8d /tidb/cdc/default/default/upstream/7375100874550838532 {"id":7375100874550838532,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03843d81-06ff-42d9-aeaf-16739fa42d8d {"id":"03843d81-06ff-42d9-aeaf-16739fa42d8d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149495} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce15ba3997 03843d81-06ff-42d9-aeaf-16739fa42d8d /tidb/cdc/default/default/upstream/7375100874550838532 {"id":7375100874550838532,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9639.out cli changefeed create --start-ts=450140436781334529 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-18470?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: ba5d5d04-912d-4e94-ab76-cfb6f197c5be Info: {"upstream_id":7375100874550838532,"namespace":"default","id":"ba5d5d04-912d-4e94-ab76-cfb6f197c5be","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-sequence-test-18470?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:58:18.320807141+08:00","start_ts":450140436781334529,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140436781334529,"checkpoint_ts":450140436781334529,"checkpoint_time":"2024-05-31 17:58:13.337"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0033f8440} {Type:update Path:[Config SyncPointRetention] From: To:0xc0033f8448} {Type:update Path:[Config Consistent] From: To:0xc0008d4070} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12659.out cli changefeed --changefeed-id custom-changefeed-name pause Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 30-th check, retry later [2024/05/31 17:58:19.309 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/31 17:58:19.311 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/31 17:58:19.312 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=dff5773b-f284-4c55-bdc3-56806bd0484a] [2024/05/31 17:58:19.312 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/31 17:58:19.313 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/31 17:58:19.447 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=be8d4912-6a98-4a11-b82c-aa0443f122b0] [captureID=033559f0-a361-4081-9b59-89d41016cea8] [processorDetail="{\"table_ids\":[]}"] [2024/05/31 17:58:19.597 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=be8d4912-6a98-4a11-b82c-aa0443f122b0] [captureID=4d71ac0c-535e-48cc-9fb8-7ef552647faa] [processorDetail="{\"table_ids\":[]}"] [2024/05/31 17:58:19.747 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=be8d4912-6a98-4a11-b82c-aa0443f122b0] [captureID=dff5773b-f284-4c55-bdc3-56806bd0484a] [processorDetail="{\"table_ids\":[108,110]}"] [2024/05/31 17:58:19.747 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"033559f0-a361-4081-9b59-89d41016cea8\":[],\"4d71ac0c-535e-48cc-9fb8-7ef552647faa\":[],\"dff5773b-f284-4c55-bdc3-56806bd0484a\":[{\"ID\":108,\"Changefeed\":\"be8d4912-6a98-4a11-b82c-aa0443f122b0\"},{\"ID\":110,\"Changefeed\":\"be8d4912-6a98-4a11-b82c-aa0443f122b0\"}]}"] [2024/05/31 17:58:19.747 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=be8d4912-6a98-4a11-b82c-aa0443f122b0&target-cp-id=4d71ac0c-535e-48cc-9fb8-7ef552647faa&table-id=108"] [2024/05/31 17:58:19.796 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/31 17:58:19.797 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=be8d4912-6a98-4a11-b82c-aa0443f122b0&target-cp-id=4d71ac0c-535e-48cc-9fb8-7ef552647faa&table-id=110"] [2024/05/31 17:58:19.847 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=110] [2024/05/31 17:58:19.847 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=dff5773b-f284-4c55-bdc3-56806bd0484a] [targetCapture=4d71ac0c-535e-48cc-9fb8-7ef552647faa] + set +x [Fri May 31 17:58:19 CST 2024] <<<<<< START kafka consumer in ddl_sequence case >>>>>> check diff successfully + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3857f16c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:19807, start at 2024-05-31 17:58:18.468930005 +0800 CST m=+2.283445937 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:18.478 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:18.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:18.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3857f16c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:19807, start at 2024-05-31 17:58:18.468930005 +0800 CST m=+2.283445937 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:18.478 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:18.459 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:18.459 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3857f30c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:19890, start at 2024-05-31 17:58:18.598627587 +0800 CST m=+2.354576088 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:18.605 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:18.663 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:18.663 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_sequence.finish_mark not exists for 1-th check, retry later table move_table.check2 not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_simple_claim_check_avro/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table mark.finish_mark_3 not exists for 31-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } table move_table.check2 exists [Pipeline] // timeout [Fri May 31 17:58:22 CST 2024] <<<<<< START cdc server in kafka_simple_claim_check_avro case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.2125821260.out server --log-file /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_simple_claim_check_avro/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container check diff successfully [Pipeline] } [Pipeline] // withEnv [Pipeline] } \033[0;36m<<< Run all test success >>>\033[0m ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 2-th check, retry later [Pipeline] // node [Pipeline] } [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) table mark.finish_mark_3 not exists for 32-th check, retry later [Pipeline] // podTemplate [Pipeline] // cache wait process cdc.test exit for 1-th time... [Pipeline] } [Pipeline] } + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450140437878931462, "checkpoint_time": "2024-05-31 17:58:17.524", "error": null }' + echo '{ "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450140437878931462, "checkpoint_time": "2024-05-31 17:58:17.524", "error": null }' { "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 450140437878931462, "checkpoint_time": "2024-05-31 17:58:17.524", "error": null } ++ echo '{' '"upstream_id":' 7375100841339931867, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450140437878931462, '"checkpoint_time":' '"2024-05-31' '17:58:17.524",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7375100841339931867, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 450140437878931462, '"checkpoint_time":' '"2024-05-31' '17:58:17.524",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12725.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name [Pipeline] // withEnv [Pipeline] // dir [Pipeline] } [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] // withCredentials [Pipeline] } [Pipeline] } Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc003a4c138} {Type:update Path:[Config SyncPointRetention] From: To:0xc003a4c148} {Type:update Path:[Config Consistent] From: To:0xc000df0540} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7375100841339931867,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-6413?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:58:11.262524872+08:00","start_ts":450140434641453057,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":true,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":0,"checkpoint_ts":450140437878931462,"checkpoint_time":"2024-05-31 17:58:17.524"} PASS coverage: 2.9% of statements in github.com/pingcap/tiflow/... [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri May 31 17:58:24 CST 2024] <<<<<< run test case move_table success! >>>>>> [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385833c00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:25478, start at 2024-05-31 17:58:22.74499275 +0800 CST m=+2.049538238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:22.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:22.755 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:22.755 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385833c00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:25478, start at 2024-05-31 17:58:22.74499275 +0800 CST m=+2.049538238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:22.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:22.755 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:22.755 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385833140006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-fdkhs-p56gc, pid:25554, start at 2024-05-31 17:58:22.668913609 +0800 CST m=+1.905261750 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:22.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:22.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:22.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_source/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_source/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_source/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table mark.finish_mark_3 not exists for 33-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12762.out cli changefeed --changefeed-id custom-changefeed-name resume + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:58:25 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28748811-b0d5-4c57-812d-f8a2337e49ee {"id":"28748811-b0d5-4c57-812d-f8a2337e49ee","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce15df3d8c 28748811-b0d5-4c57-812d-f8a2337e49ee /tidb/cdc/default/default/upstream/7375100914535602358 {"id":7375100914535602358,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28748811-b0d5-4c57-812d-f8a2337e49ee {"id":"28748811-b0d5-4c57-812d-f8a2337e49ee","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce15df3d8c 28748811-b0d5-4c57-812d-f8a2337e49ee /tidb/cdc/default/default/upstream/7375100914535602358 {"id":7375100914535602358,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/28748811-b0d5-4c57-812d-f8a2337e49ee {"id":"28748811-b0d5-4c57-812d-f8a2337e49ee","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149502} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce15df3d8c 28748811-b0d5-4c57-812d-f8a2337e49ee /tidb/cdc/default/default/upstream/7375100914535602358 {"id":7375100914535602358,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.21316.out cli tso query --pd=http://127.0.0.1:2379 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table ddl_sequence.finish_mark not exists for 3-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.26934.out cli tso query --pd=http://127.0.0.1:2379 table mark.finish_mark_3 not exists for 34-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:58:27.230 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 2 another is 4"] + set +x + set +x + tso='450140440116592642 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140440116592642 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.21349.out cli changefeed create --start-ts=450140440116592642 '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-14590?protocol=simple&encoding-format=avro' -c kafka-simple-claim-check-avro --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/conf/changefeed.toml Create changefeed successfully! ID: kafka-simple-claim-check-avro Info: {"upstream_id":7375100914535602358,"namespace":"default","id":"kafka-simple-claim-check-avro","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-14590?protocol=simple\u0026encoding-format=avro","create_time":"2024-05-31T17:58:27.929082876+08:00","start_ts":450140440116592642,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-avro-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140440116592642,"checkpoint_ts":450140440116592642,"checkpoint_time":"2024-05-31 17:58:26.060"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table ddl_sequence.finish_mark not exists for 4-th check, retry later + set +x + tso='450140440442699777 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140440442699777 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:58:28 CST 2024] <<<<<< START cdc server in multi_source case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.2696326965.out server --log-file /tmp/tidb_cdc_test/multi_source/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_source/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table mark.finish_mark_3 not exists for 35-th check, retry later + set +x table ddl_sequence.finish_mark not exists for 5-th check, retry later table mark.finish_mark_3 not exists for 36-th check, retry later + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140440762253323, "checkpoint_time": "2024-05-31 17:58:28.523", "error": null }' + echo '{ "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140440762253323, "checkpoint_time": "2024-05-31 17:58:28.523", "error": null }' { "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140440762253323, "checkpoint_time": "2024-05-31 17:58:28.523", "error": null } ++ echo '{' '"upstream_id":' 7375100841339931867, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140440762253323, '"checkpoint_time":' '"2024-05-31' '17:58:28.523",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375100841339931867, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140440762253323, '"checkpoint_time":' '"2024-05-31' '17:58:28.523",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12826.out cli changefeed --changefeed-id custom-changefeed-name remove Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 450140441286803469 SinkURI: kafka://127.0.0.1:9092/ticdc-cli-test-6413?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760 PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:58:31 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aa57b954-c608-4377-86e5-3bbc83548eb5 {"id":"aa57b954-c608-4377-86e5-3bbc83548eb5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149509} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce15ece49a aa57b954-c608-4377-86e5-3bbc83548eb5 /tidb/cdc/default/default/upstream/7375100924042656641 {"id":7375100924042656641,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aa57b954-c608-4377-86e5-3bbc83548eb5 {"id":"aa57b954-c608-4377-86e5-3bbc83548eb5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149509} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce15ece49a aa57b954-c608-4377-86e5-3bbc83548eb5 /tidb/cdc/default/default/upstream/7375100924042656641 {"id":7375100924042656641,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aa57b954-c608-4377-86e5-3bbc83548eb5 {"id":"aa57b954-c608-4377-86e5-3bbc83548eb5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149509} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce15ece49a aa57b954-c608-4377-86e5-3bbc83548eb5 /tidb/cdc/default/default/upstream/7375100924042656641 {"id":7375100924042656641,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_source.cli.27015.out cli changefeed create --start-ts=450140440442699777 '--sink-uri=kafka://127.0.0.1:9092/ticdc-multi-source-test-13566?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: d1eb61dd-6ec5-4f57-95ba-190e8571c2f2 Info: {"upstream_id":7375100924042656641,"namespace":"default","id":"d1eb61dd-6ec5-4f57-95ba-190e8571c2f2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-source-test-13566?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:58:32.331169986+08:00","start_ts":450140440442699777,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140440442699777,"checkpoint_ts":450140440442699777,"checkpoint_time":"2024-05-31 17:58:27.304"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_sequence.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... wait process cdc.test exit for 1-th time... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table mark.finish_mark_3 not exists for 37-th check, retry later wait process cdc.test exit for 2-th time... + set +x [Fri May 31 17:58:33 CST 2024] <<<<<< START kafka consumer in multi_source case >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:58:33 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.21395.out cli changefeed pause -c kafka-simple-claim-check-avro ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 38-th check, retry later changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12887.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-cli-test-6413?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.21425.out cli changefeed update -c kafka-simple-claim-check-avro '--sink-uri=kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-14590?protocol=simple&encoding-format=avro&max-message-bytes=2048' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/conf/changefeed.toml --no-confirm Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7375100841339931867,"namespace":"default","id":"custom-changefeed-name","sink_uri":"kafka://127.0.0.1:9092/ticdc-cli-test-6413?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:58:36.535216698+08:00","start_ts":450140442820083716,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140442820083716,"checkpoint_ts":450140442820083716,"checkpoint_time":"2024-05-31 17:58:36.373"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Diff of changefeed config: {Type:update Path:[SinkURI] From:kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-14590?protocol=simple&encoding-format=avro To:kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-14590?protocol=simple&encoding-format=avro&max-message-bytes=2048} {Type:update Path:[Config SyncPointInterval] From: To:0xc0036434f8} {Type:update Path:[Config SyncPointRetention] From: To:0xc003643508} {Type:update Path:[Config Consistent] From: To:0xc00001f5e0} Update changefeed config successfully! ID: kafka-simple-claim-check-avro Info: {"upstream_id":7375100914535602358,"namespace":"default","id":"kafka-simple-claim-check-avro","sink_uri":"kafka://127.0.0.1:9092/kafka-simple-claim-check-avro-14590?protocol=simple\u0026encoding-format=avro\u0026max-message-bytes=2048","create_time":"2024-05-31T17:58:27.929082876+08:00","start_ts":450140440116592642,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"simple","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/kafka-simple-avro-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"error"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":0,"checkpoint_ts":450140442253066244,"checkpoint_time":"2024-05-31 17:58:34.210"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 39-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3858efac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k, pid:24525, start at 2024-05-31 17:58:34.758891082 +0800 CST m=+2.167473531 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:34.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:34.731 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:34.731 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3858efac0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k, pid:24525, start at 2024-05-31 17:58:34.758891082 +0800 CST m=+2.167473531 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:34.771 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:34.731 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:34.731 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3858f06c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-qppb5-00f0k, pid:24596, start at 2024-05-31 17:58:34.801733086 +0800 CST m=+2.142806178 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:34.808 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:34.779 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:34.779 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_simple_claim_check_avro.cli.21446.out cli changefeed resume -c kafka-simple-claim-check-avro [2024/05/31 17:58:38.501 +08:00] [INFO] [main.go:86] ["running ddl test: 0 createDropSchemaDDL"] PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table mark.finish_mark_3 not exists for 40-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.25978.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:58:39.740 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] + set +x table test.finish_mark not exists for 1-th check, retry later table mark.finish_mark_3 not exists for 41-th check, retry later + set +x + tso='450140443608875009 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140443608875009 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:58:40 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.2600526007.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140443921350659, "checkpoint_time": "2024-05-31 17:58:40.574", "error": null }' + echo '{ "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140443921350659, "checkpoint_time": "2024-05-31 17:58:40.574", "error": null }' { "upstream_id": 7375100841339931867, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 450140443921350659, "checkpoint_time": "2024-05-31 17:58:40.574", "error": null } ++ echo '{' '"upstream_id":' 7375100841339931867, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140443921350659, '"checkpoint_time":' '"2024-05-31' '17:58:40.574",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375100841339931867, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 450140443921350659, '"checkpoint_time":' '"2024-05-31' '17:58:40.574",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.12973.out cli changefeed create --start-ts=450140434641453057 '--sink-uri=kafka://127.0.0.1:9093/ticdc-cli-test-ssl-17262?protocol=open-protocol&ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem&cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem&key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem&kafka-version=2.4.1&max-message-bytes=10485760&insecure-skip-verify=true' --tz=Asia/Shanghai [2024/05/31 17:58:41.953 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:58:41.958 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: 41f8aef0-6a72-498c-ab9d-6316b1439055 Info: {"upstream_id":7375100841339931867,"namespace":"default","id":"41f8aef0-6a72-498c-ab9d-6316b1439055","sink_uri":"kafka://127.0.0.1:9093/ticdc-cli-test-ssl-17262?protocol=open-protocol\u0026ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/ca.pem\u0026cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client.pem\u0026key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_certificates/client-key.pem\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760\u0026insecure-skip-verify=true","create_time":"2024-05-31T17:58:42.557492785+08:00","start_ts":450140434641453057,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140434641453057,"checkpoint_ts":450140434641453057,"checkpoint_time":"2024-05-31 17:58:05.174"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... 5:58PM INF > Info cdc.mysql=kafka://127.0.0.1:9092/output_debezium cdc.tidb=kafka://127.0.0.1:9092/output_ticdc db.mysql=root@tcp(127.0.0.1:3310)/{db}?allowNativePasswords=true db.tidb=root@tcp(127.0.0.1:4000)/{db}?allowNativePasswords=true 5:58PM INF > Run case=sql/data_types.sql table test.finish_mark not exists for 2-th check, retry later [2024/05/31 17:58:43.170 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] table mark.finish_mark_3 not exists for 42-th check, retry later [mysql] 2024/05/31 17:58:43 connection.go:299: invalid connection + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13004.out cli unsafe delete-service-gc-safepoint + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:58:43 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88c9beeb-49c3-4fb9-b727-79eadd7b6c47 {"id":"88c9beeb-49c3-4fb9-b727-79eadd7b6c47","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce161da095 88c9beeb-49c3-4fb9-b727-79eadd7b6c47 /tidb/cdc/default/default/upstream/7375100984122845646 {"id":7375100984122845646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88c9beeb-49c3-4fb9-b727-79eadd7b6c47 {"id":"88c9beeb-49c3-4fb9-b727-79eadd7b6c47","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce161da095 88c9beeb-49c3-4fb9-b727-79eadd7b6c47 /tidb/cdc/default/default/upstream/7375100984122845646 {"id":7375100984122845646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/88c9beeb-49c3-4fb9-b727-79eadd7b6c47 {"id":"88c9beeb-49c3-4fb9-b727-79eadd7b6c47","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149521} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce161da095 88c9beeb-49c3-4fb9-b727-79eadd7b6c47 /tidb/cdc/default/default/upstream/7375100984122845646 {"id":7375100984122845646,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.26059.out cli changefeed create --start-ts=450140443608875009 '--sink-uri=kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-9317?protocol=open-protocol&partition-num=4&kafka-version=2.4.1' Confirm that you know what this command will do and use it at your own risk [Y/N] Create changefeed successfully! ID: a1711f8e-6885-44a0-8878-2b5ab2175935 Info: {"upstream_id":7375100984122845646,"namespace":"default","id":"a1711f8e-6885-44a0-8878-2b5ab2175935","sink_uri":"kafka://127.0.0.1:9092/ticdc-ddl-attributes-test-9317?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1","create_time":"2024-05-31T17:58:44.451421782+08:00","start_ts":450140443608875009,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140443608875009,"checkpoint_ts":450140443608875009,"checkpoint_time":"2024-05-31 17:58:39.382"} PASS CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table mark.finish_mark_3 exists table mark.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13038.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x [Fri May 31 17:58:45 CST 2024] <<<<<< START kafka consumer in ddl_attributes case >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:58:46 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> reset and all metadata truncated in PD! PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table mark.finish_mark exists check diff successfully + set +x Verifying downstream PD is started... wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... [2024/05/31 17:58:48.517 +08:00] [INFO] [main.go:86] ["running ddl test: 1 truncateDDL"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:58:48 CST 2024] <<<<<< run test case default_value success! >>>>>> Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:58:49.229 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:58:49.261 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:58:49.409 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:58:49.415 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:58:49.434 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:58:49.436 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:58:49.588 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:58:49.612 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 17:58:49.774 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:58:49.780 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:58:49.798 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:58:49.807 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:58:49.969 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 17:58:49.982 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 17:58:50.138 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:58:50.143 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:58:50.149 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:58:50.157 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] table ddl_attributes.attributes_t1_new not exists for 1-th check, retry later [2024/05/31 17:58:50.323 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 17:58:50.352 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:58:50.516 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 17:58:50.525 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:58:50.555 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:58:50.564 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:58:50.693 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 17:58:50.735 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13124.out cli unsafe resolve-lock --region=78 [2024/05/31 17:58:50.871 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:58:50.880 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 17:58:50.914 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 17:58:50.922 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... [2024/05/31 17:58:51.072 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 17:58:51.108 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 17:58:51.654 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 17:58:51.666 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:58:51.683 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 17:58:51.692 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 17:58:51.823 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 17:58:51.854 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 17:58:51.991 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 17:58:52.003 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:58:52.024 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:58:52.033 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] table ddl_attributes.attributes_t1_new not exists for 2-th check, retry later [2024/05/31 17:58:52.173 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 17:58:52.198 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13149.out cli unsafe resolve-lock --region=78 --ts=450140445769728006 [2024/05/31 17:58:52.346 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 17:58:52.361 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 17:58:52.375 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 17:58:52.385 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 17:58:52.519 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 17:58:52.548 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [2024/05/31 17:58:52.687 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 17:58:52.702 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 17:58:52.720 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 17:58:52.729 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [2024/05/31 17:58:52.865 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 17:58:52.969 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/31 17:58:53.038 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/31 17:58:53.054 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 17:58:53.144 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 17:58:53.153 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 17:58:53.212 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 17:58:53.322 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 17:58:53.389 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 17:58:53.404 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 17:58:53.497 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/31 17:58:53.505 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385a1b40000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:30369, start at 2024-05-31 17:58:53.916336168 +0800 CST m=+1.913157147 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:53.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:53.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:53.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385a1b40000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:30369, start at 2024-05-31 17:58:53.916336168 +0800 CST m=+1.913157147 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:53.923 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:53.904 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:53.904 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385a1c000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:30445, start at 2024-05-31 17:58:53.997688563 +0800 CST m=+1.933830059 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:54.005 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:54.002 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:54.002 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 138k 4010 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore table ddl_attributes.attributes_t1_new not exists for 3-th check, retry later table ddl_attributes.attributes_t1_new not exists for 4-th check, retry later [Fri May 31 17:58:56 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3174931751.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 199k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v8.2.0-alpha-75-g18cf66054", "git_hash": "18cf66054053b7e9a688825ac1a73321e0b6ee8d", "id": "69c0be20-4b84-4a4a-aadf-a2e4fa3b8880", "pid": 12310, "is_owner": true }wait process cdc.test exit for 1-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh: line 1: 21480 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table ddl_attributes.attributes_t1_new not exists for 5-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:58:58 CST 2024] <<<<<< run test case cli_with_auth success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:58:59 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c49728e8-af25-454b-9a1b-5c61f06ee410 {"id":"c49728e8-af25-454b-9a1b-5c61f06ee410","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149536} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16643985 c49728e8-af25-454b-9a1b-5c61f06ee410 /tidb/cdc/default/default/upstream/7375101056670109751 {"id":7375101056670109751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c49728e8-af25-454b-9a1b-5c61f06ee410 {"id":"c49728e8-af25-454b-9a1b-5c61f06ee410","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149536} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16643985 c49728e8-af25-454b-9a1b-5c61f06ee410 /tidb/cdc/default/default/upstream/7375101056670109751 {"id":7375101056670109751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c49728e8-af25-454b-9a1b-5c61f06ee410 {"id":"c49728e8-af25-454b-9a1b-5c61f06ee410","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149536} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16643985 c49728e8-af25-454b-9a1b-5c61f06ee410 /tidb/cdc/default/default/upstream/7375101056670109751 {"id":7375101056670109751,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 3922a248-6207-42de-84e9-1cc8468c701c Info: {"upstream_id":7375101056670109751,"namespace":"default","id":"3922a248-6207-42de-84e9-1cc8468c701c","sink_uri":"kafka://127.0.0.1:9092/ticdc-autorandom-test-16637?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:58:59.629879696+08:00","start_ts":450140448883736578,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140448883736578,"checkpoint_ts":450140448883736578,"checkpoint_time":"2024-05-31 17:58:59.504"} [Fri May 31 17:58:59 CST 2024] <<<<<< START kafka consumer in autorandom case >>>>>> [2024/05/31 17:58:59.045 +08:00] [INFO] [main.go:86] ["running ddl test: 2 addDropColumnDDL"] [2024/05/31 17:58:59.247 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:58:59.268 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:58:59.424 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:58:59.426 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:58:59.443 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:58:59.444 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:58:59.594 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:58:59.619 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 17:58:59.773 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:58:59.775 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:58:59.802 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:58:59.803 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:58:59.965 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 17:58:59.999 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later [2024/05/31 17:59:00.164 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:59:00.166 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:59:00.207 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 17:59:00.208 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:59:00.393 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 17:59:00.441 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:59:00.599 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 17:59:00.601 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:59:00.646 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:59:00.647 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:59:00.809 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 17:59:00.860 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] table autorandom_test.table_a not exists for 1-th check, retry later [2024/05/31 17:59:01.017 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:59:01.019 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 17:59:01.067 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 17:59:01.068 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/31 17:59:01.243 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 17:59:01.294 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385a68340013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:10889, start at 2024-05-31 17:58:58.855980464 +0800 CST m=+2.050551551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:00:58.865 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:58.829 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:58.829 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:59:01.457 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 17:59:01.459 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:59:01.506 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 17:59:01.508 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] start tidb cluster in /tmp/tidb_cdc_test/canal_json_adapter_compatibility Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:59:01.676 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 17:59:01.725 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 17:59:01.887 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 17:59:01.889 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:59:01.935 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/31 17:59:01.936 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:59:02.097 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 17:59:02.141 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] table ddl_attributes.finish_mark not exists for 2-th check, retry later [2024/05/31 17:59:02.291 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 17:59:02.293 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 17:59:02.338 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 17:59:02.339 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] table autorandom_test.table_a exists check diff failed 1-th time, retry later [2024/05/31 17:59:02.715 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 17:59:02.825 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 17:59:02.962 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 17:59:02.964 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 17:59:03.016 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 17:59:03.018 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 17:59:03.158 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 17:59:03.210 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:59:03.345 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/31 17:59:03.346 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 17:59:03.395 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 17:59:03.396 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:59:03.533 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 17:59:03.593 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 17:59:03.720 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 17:59:03.722 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 17:59:03.774 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/31 17:59:03.775 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/31 17:59:03.907 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/31 17:59:03.960 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_attributes.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... Verifying downstream PD is started... wait process cdc.test exit for 2-th time... table ddl_attributes.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:59:06 CST 2024] <<<<<< run test case autorandom success! >>>>>> Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_attributes.finish_mark not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385affec0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:22695, start at 2024-05-31 17:59:08.540141504 +0800 CST m=+1.977019976 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:08.547 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:08.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:08.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385affec0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:22695, start at 2024-05-31 17:59:08.540141504 +0800 CST m=+1.977019976 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:08.547 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:08.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:08.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385afef80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:22773, start at 2024-05-31 17:59:08.511027466 +0800 CST m=+1.878251031 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:08.518 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:08.478 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:08.478 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_adapter_compatibility/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_attributes.finish_mark not exists for 6-th check, retry later [2024/05/31 17:59:10.213 +08:00] [INFO] [main.go:86] ["running ddl test: 3 addDropColumnDDL2"] [2024/05/31 17:59:10.424 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:59:10.445 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:59:10.609 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:59:10.610 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:59:10.637 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:59:10.639 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.24074.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:59:10.806 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:59:10.835 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 17:59:11.007 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:59:11.008 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385b1ef80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:19682, start at 2024-05-31 17:59:10.559872797 +0800 CST m=+1.810979968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:10.566 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:10.526 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:10.526 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385b1ef80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:19682, start at 2024-05-31 17:59:10.559872797 +0800 CST m=+1.810979968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:10.566 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:10.526 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:10.526 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385b1d440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:19754, start at 2024-05-31 17:59:10.452988726 +0800 CST m=+1.652986218 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:10.459 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:10.417 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:10.417 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/31 17:59:11.036 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:59:11.038 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:59:11.201 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 17:59:11.229 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:59:11.399 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:59:11.400 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:59:11.426 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:59:11.428 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 17:59:11.585 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 17:59:11.612 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:59:11.763 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:59:11.764 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 17:59:11.791 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:59:11.793 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:59:11.944 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 17:59:11.971 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 17:59:12.142 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 17:59:12.143 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:59:12.169 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 17:59:12.171 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] table ddl_attributes.finish_mark not exists for 7-th check, retry later [2024/05/31 17:59:12.329 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 17:59:12.351 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 17:59:12.504 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:59:12.505 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 17:59:12.529 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 17:59:12.531 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 17:59:12.695 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 17:59:12.724 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] + set +x + tso='450140451947151361 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140451947151361 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:59:12 CST 2024] <<<<<< START cdc server in canal_json_adapter_compatibility case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.2410524107.out server --log-file /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_adapter_compatibility/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.21111.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:59:12.886 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:59:12.887 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 17:59:12.912 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:59:12.913 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:59:13.073 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 17:59:13.111 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 17:59:13.257 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 17:59:13.258 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 17:59:13.303 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 17:59:13.305 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 17:59:13.446 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 17:59:13.498 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 17:59:13.628 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 17:59:13.629 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 17:59:13.681 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 17:59:13.683 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:59:14.123 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 17:59:14.168 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] [2024/05/31 17:59:14.317 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 17:59:14.318 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/31 17:59:14.374 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 17:59:14.376 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 17:59:14.509 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 17:59:14.581 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] table ddl_attributes.finish_mark not exists for 8-th check, retry later [2024/05/31 17:59:14.717 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 17:59:14.718 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 17:59:14.803 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/31 17:59:14.805 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] + set +x + tso='450140452507090945 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140452507090945 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [2024/05/31 17:59:14.930 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/31 17:59:15.010 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/31 17:59:15.131 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/31 17:59:15.132 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Fri May 31 17:59:15 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.2115321155.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:59:15 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb603323-efbc-4424-a567-a429d8286719 {"id":"cb603323-efbc-4424-a567-a429d8286719","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16a43a88 cb603323-efbc-4424-a567-a429d8286719 /tidb/cdc/default/default/upstream/7375101129171520317 {"id":7375101129171520317,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb603323-efbc-4424-a567-a429d8286719 {"id":"cb603323-efbc-4424-a567-a429d8286719","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16a43a88 cb603323-efbc-4424-a567-a429d8286719 /tidb/cdc/default/default/upstream/7375101129171520317 {"id":7375101129171520317,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cb603323-efbc-4424-a567-a429d8286719 {"id":"cb603323-efbc-4424-a567-a429d8286719","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16a43a88 cb603323-efbc-4424-a567-a429d8286719 /tidb/cdc/default/default/upstream/7375101129171520317 {"id":7375101129171520317,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_adapter_compatibility.cli.24152.out cli changefeed create --start-ts=450140451947151361 '--sink-uri=kafka://127.0.0.1:9092/test?protocol=canal-json&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 3de2a17a-db62-4a12-b74b-ab9a7678eb51 Info: {"upstream_id":7375101129171520317,"namespace":"default","id":"3de2a17a-db62-4a12-b74b-ab9a7678eb51","sink_uri":"kafka://127.0.0.1:9092/test?protocol=canal-json\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:59:16.175877869+08:00","start_ts":450140451947151361,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140451947151361,"checkpoint_ts":450140451947151361,"checkpoint_time":"2024-05-31 17:59:11.190"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_attributes.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:59:18 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:59:18 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66f0a294-7870-4f4e-bc6a-fffe5c395c6b {"id":"66f0a294-7870-4f4e-bc6a-fffe5c395c6b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149556} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16acc5a6 66f0a294-7870-4f4e-bc6a-fffe5c395c6b /tidb/cdc/default/default/upstream/7375101137584392919 {"id":7375101137584392919,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66f0a294-7870-4f4e-bc6a-fffe5c395c6b {"id":"66f0a294-7870-4f4e-bc6a-fffe5c395c6b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149556} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16acc5a6 66f0a294-7870-4f4e-bc6a-fffe5c395c6b /tidb/cdc/default/default/upstream/7375101137584392919 {"id":7375101137584392919,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66f0a294-7870-4f4e-bc6a-fffe5c395c6b {"id":"66f0a294-7870-4f4e-bc6a-fffe5c395c6b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149556} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16acc5a6 66f0a294-7870-4f4e-bc6a-fffe5c395c6b /tidb/cdc/default/default/upstream/7375101137584392919 {"id":7375101137584392919,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.21212.out cli changefeed create --start-ts=450140452507090945 '--sink-uri=kafka+ssl://127.0.0.1:9092/ticdc-simple-test-23548?protocol=open-protocol&partition-num=4&kafka-client-id=cdc_test_simple&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: f3a78c9c-e96f-458d-9637-d39767bccea2 Info: {"upstream_id":7375101137584392919,"namespace":"default","id":"f3a78c9c-e96f-458d-9637-d39767bccea2","sink_uri":"kafka+ssl://127.0.0.1:9092/ticdc-simple-test-23548?protocol=open-protocol\u0026partition-num=4\u0026kafka-client-id=cdc_test_simple\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:59:19.266310779+08:00","start_ts":450140452507090945,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140452507090945,"checkpoint_ts":450140452507090945,"checkpoint_time":"2024-05-31 17:59:13.326"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 1-th check, retry later [2024/05/31 17:59:20.240 +08:00] [INFO] [main.go:86] ["running ddl test: 4 modifyColumnDDL"] [2024/05/31 17:59:20.457 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:59:20.481 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] + set +x [Fri May 31 17:59:20 CST 2024] <<<<<< START kafka consumer in simple case >>>>>> succeed to verify meta placement rules [2024/05/31 17:59:20.639 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:59:20.641 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:59:20.670 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:59:20.672 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:59:20.827 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:59:20.853 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later [2024/05/31 17:59:21.015 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:59:21.017 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:59:21.040 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:59:21.042 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:59:21.210 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] [2024/05/31 17:59:21.232 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:59:21.383 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:59:21.385 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:59:21.404 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:59:21.406 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 17:59:21.561 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 17:59:21.586 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:59:21.741 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 17:59:21.743 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] [2024/05/31 17:59:21.764 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:59:21.766 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385bcd8c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:14240, start at 2024-05-31 17:59:21.735071585 +0800 CST m=+1.839432565 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:21.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:21.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:21.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385bcd8c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:14240, start at 2024-05-31 17:59:21.735071585 +0800 CST m=+1.839432565 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:21.741 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:21.699 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:21.699 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385bcca40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:14313, start at 2024-05-31 17:59:21.679438538 +0800 CST m=+1.731577587 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:21.686 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:21.691 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:21.691 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table test.binary_columns not exists for 2-th check, retry later [2024/05/31 17:59:21.923 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 17:59:21.940 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] [2024/05/31 17:59:22.120 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:59:22.122 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 17:59:22.131 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 17:59:22.133 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [2024/05/31 17:59:22.296 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 17:59:22.313 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 17:59:22.469 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 17:59:22.470 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:59:22.485 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 17:59:22.487 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] [2024/05/31 17:59:22.636 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 2-th time, retry later [2024/05/31 17:59:22.653 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] [2024/05/31 17:59:22.830 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 17:59:22.831 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:59:22.848 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:59:22.850 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] start tidb cluster in /tmp/tidb_cdc_test/generate_column Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:59:23.012 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 17:59:23.032 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:59:23.186 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 17:59:23.188 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] [2024/05/31 17:59:23.210 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 17:59:23.212 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 17:59:23.371 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 17:59:23.394 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] [2024/05/31 17:59:14.478 +08:00] [INFO] [case.go:115] ["sync updatePKUK take: 11.30514928s"] [2024/05/31 17:59:23.547 +08:00] [INFO] [main.go:220] ["0 insert success: 1800"] [2024/05/31 17:59:23.549 +08:00] [INFO] [main.go:234] ["0 delete success: 900"] [2024/05/31 17:59:23.565 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 17:59:23.567 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 17:59:23.733 +08:00] [INFO] [main.go:220] ["0 insert success: 1900"] [2024/05/31 17:59:23.745 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] table test.binary_columns not exists for 3-th check, retry later [2024/05/31 17:59:23.916 +08:00] [INFO] [main.go:220] ["0 insert success: 2000"] [2024/05/31 17:59:23.918 +08:00] [INFO] [main.go:234] ["0 delete success: 1000"] [2024/05/31 17:59:23.929 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 17:59:23.931 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] [2024/05/31 17:59:24.089 +08:00] [INFO] [main.go:220] ["0 insert success: 2100"] [2024/05/31 17:59:24.095 +08:00] [INFO] [main.go:220] ["1 insert success: 2100"] [2024/05/31 17:59:24.261 +08:00] [INFO] [main.go:220] ["0 insert success: 2200"] [2024/05/31 17:59:24.263 +08:00] [INFO] [main.go:234] ["0 delete success: 1100"] [2024/05/31 17:59:24.269 +08:00] [INFO] [main.go:220] ["1 insert success: 2200"] [2024/05/31 17:59:24.271 +08:00] [INFO] [main.go:234] ["1 delete success: 1100"] [2024/05/31 17:59:24.435 +08:00] [INFO] [main.go:220] ["0 insert success: 2300"] [2024/05/31 17:59:24.445 +08:00] [INFO] [main.go:220] ["1 insert success: 2300"] [2024/05/31 17:59:24.606 +08:00] [INFO] [main.go:220] ["0 insert success: 2400"] [2024/05/31 17:59:24.608 +08:00] [INFO] [main.go:234] ["0 delete success: 1200"] [2024/05/31 17:59:24.619 +08:00] [INFO] [main.go:220] ["1 insert success: 2400"] [2024/05/31 17:59:24.621 +08:00] [INFO] [main.go:234] ["1 delete success: 1200"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15688.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:59:24.781 +08:00] [INFO] [main.go:220] ["0 insert success: 2500"] [2024/05/31 17:59:24.795 +08:00] [INFO] [main.go:220] ["1 insert success: 2500"] [2024/05/31 17:59:24.954 +08:00] [INFO] [main.go:220] ["0 insert success: 2600"] [2024/05/31 17:59:24.956 +08:00] [INFO] [main.go:234] ["0 delete success: 1300"] [2024/05/31 17:59:24.967 +08:00] [INFO] [main.go:220] ["1 insert success: 2600"] [2024/05/31 17:59:24.969 +08:00] [INFO] [main.go:234] ["1 delete success: 1300"] TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 4-th check, retry later + set +x + tso='450140455436025857 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140455436025857 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:59:25 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1572015722.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:59:28 CST 2024] <<<<<< run test case simple success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:59:29 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fafb340-3a58-414b-9cc1-ca8ab49521c3 {"id":"9fafb340-3a58-414b-9cc1-ca8ab49521c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149566} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16d48185 9fafb340-3a58-414b-9cc1-ca8ab49521c3 /tidb/cdc/default/default/upstream/7375101188572414729 {"id":7375101188572414729,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fafb340-3a58-414b-9cc1-ca8ab49521c3 {"id":"9fafb340-3a58-414b-9cc1-ca8ab49521c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149566} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16d48185 9fafb340-3a58-414b-9cc1-ca8ab49521c3 /tidb/cdc/default/default/upstream/7375101188572414729 {"id":7375101188572414729,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9fafb340-3a58-414b-9cc1-ca8ab49521c3 {"id":"9fafb340-3a58-414b-9cc1-ca8ab49521c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149566} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16d48185 9fafb340-3a58-414b-9cc1-ca8ab49521c3 /tidb/cdc/default/default/upstream/7375101188572414729 {"id":7375101188572414729,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.15774.out cli changefeed create --start-ts=450140455436025857 '--sink-uri=kafka://127.0.0.1:9092/ticdc-savepoint-test-7303?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' Create changefeed successfully! ID: 6138ad57-ab68-4ba9-935f-217cba4ef9f7 Info: {"upstream_id":7375101188572414729,"namespace":"default","id":"6138ad57-ab68-4ba9-935f-217cba4ef9f7","sink_uri":"kafka://127.0.0.1:9092/ticdc-savepoint-test-7303?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:59:29.475543159+08:00","start_ts":450140455436025857,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140455436025857,"checkpoint_ts":450140455436025857,"checkpoint_time":"2024-05-31 17:59:24.499"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 6-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385c5a580007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:33085, start at 2024-05-31 17:59:30.716090554 +0800 CST m=+1.957471848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:30.725 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:30.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:30.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385c5a580007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:33085, start at 2024-05-31 17:59:30.716090554 +0800 CST m=+1.957471848 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:30.725 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:30.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:30.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385c5a540003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-js4zt-x992f, pid:33161, start at 2024-05-31 17:59:30.712404603 +0800 CST m=+1.897973265 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:30.720 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:30.709 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:30.709 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... [Pipeline] // container + set +x [Fri May 31 17:59:30 CST 2024] <<<<<< START kafka consumer in savepoint case >>>>>> [Pipeline] } [Pipeline] // withEnv [Pipeline] } Logging trace to /tmp/tidb_cdc_test/generate_column/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/generate_column/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/generate_column/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [2024/05/31 17:59:30.877 +08:00] [INFO] [main.go:86] ["running ddl test: 5 addDropIndexDDL"] [2024/05/31 17:59:31.074 +08:00] [INFO] [main.go:220] ["1 insert success: 100"] [2024/05/31 17:59:31.075 +08:00] [INFO] [main.go:220] ["0 insert success: 100"] [2024/05/31 17:59:31.234 +08:00] [INFO] [main.go:220] ["1 insert success: 200"] [2024/05/31 17:59:31.235 +08:00] [INFO] [main.go:220] ["0 insert success: 200"] [2024/05/31 17:59:31.236 +08:00] [INFO] [main.go:234] ["1 delete success: 100"] [2024/05/31 17:59:31.238 +08:00] [INFO] [main.go:234] ["0 delete success: 100"] [2024/05/31 17:59:31.400 +08:00] [INFO] [main.go:220] ["1 insert success: 300"] [2024/05/31 17:59:31.413 +08:00] [INFO] [main.go:220] ["0 insert success: 300"] [2024/05/31 17:59:31.583 +08:00] [INFO] [main.go:220] ["1 insert success: 400"] [2024/05/31 17:59:31.585 +08:00] [INFO] [main.go:234] ["1 delete success: 200"] [2024/05/31 17:59:31.602 +08:00] [INFO] [main.go:220] ["0 insert success: 400"] [2024/05/31 17:59:31.604 +08:00] [INFO] [main.go:234] ["0 delete success: 200"] [2024/05/31 17:59:31.839 +08:00] [INFO] [main.go:220] ["1 insert success: 500"] [2024/05/31 17:59:31.854 +08:00] [INFO] [main.go:220] ["0 insert success: 500"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 7-th check, retry later [2024/05/31 17:59:32.089 +08:00] [INFO] [main.go:220] ["1 insert success: 600"] [2024/05/31 17:59:32.091 +08:00] [INFO] [main.go:234] ["1 delete success: 300"] [2024/05/31 17:59:32.095 +08:00] [INFO] [main.go:220] ["0 insert success: 600"] [2024/05/31 17:59:32.097 +08:00] [INFO] [main.go:234] ["0 delete success: 300"] [2024/05/31 17:59:32.339 +08:00] [INFO] [main.go:220] ["1 insert success: 700"] [2024/05/31 17:59:32.349 +08:00] [INFO] [main.go:220] ["0 insert success: 700"] [2024/05/31 17:59:32.600 +08:00] [INFO] [main.go:220] ["1 insert success: 800"] [2024/05/31 17:59:32.605 +08:00] [INFO] [main.go:234] ["1 delete success: 400"] [2024/05/31 17:59:32.629 +08:00] [INFO] [main.go:220] ["0 insert success: 800"] [2024/05/31 17:59:32.632 +08:00] [INFO] [main.go:234] ["0 delete success: 400"] table savepoint.finish_mark not exists for 1-th check, retry later [2024/05/31 17:59:32.921 +08:00] [INFO] [main.go:220] ["1 insert success: 900"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34486.out cli tso query --pd=http://127.0.0.1:2379 [2024/05/31 17:59:33.041 +08:00] [INFO] [main.go:220] ["0 insert success: 900"] [2024/05/31 17:59:33.223 +08:00] [INFO] [main.go:220] ["1 insert success: 1000"] [2024/05/31 17:59:33.226 +08:00] [INFO] [main.go:234] ["1 delete success: 500"] [2024/05/31 17:59:33.489 +08:00] [INFO] [main.go:220] ["0 insert success: 1000"] [2024/05/31 17:59:33.493 +08:00] [INFO] [main.go:234] ["0 delete success: 500"] [2024/05/31 17:59:33.538 +08:00] [INFO] [main.go:220] ["1 insert success: 1100"] [2024/05/31 17:59:33.757 +08:00] [INFO] [main.go:220] ["1 insert success: 1200"] [2024/05/31 17:59:33.759 +08:00] [INFO] [main.go:234] ["1 delete success: 600"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/31 17:59:33.808 +08:00] [INFO] [main.go:220] ["0 insert success: 1100"] [2024/05/31 17:59:33.934 +08:00] [INFO] [main.go:220] ["1 insert success: 1300"] table test.binary_columns not exists for 8-th check, retry later [2024/05/31 17:59:34.069 +08:00] [INFO] [main.go:220] ["0 insert success: 1200"] [2024/05/31 17:59:34.072 +08:00] [INFO] [main.go:234] ["0 delete success: 600"] [2024/05/31 17:59:34.121 +08:00] [INFO] [main.go:220] ["1 insert success: 1400"] [2024/05/31 17:59:34.123 +08:00] [INFO] [main.go:234] ["1 delete success: 700"] [2024/05/31 17:59:34.302 +08:00] [INFO] [main.go:220] ["1 insert success: 1500"] [2024/05/31 17:59:34.329 +08:00] [INFO] [main.go:220] ["0 insert success: 1300"] [2024/05/31 17:59:34.486 +08:00] [INFO] [main.go:220] ["1 insert success: 1600"] [2024/05/31 17:59:34.488 +08:00] [INFO] [main.go:234] ["1 delete success: 800"] [2024/05/31 17:59:34.582 +08:00] [INFO] [main.go:220] ["0 insert success: 1400"] [2024/05/31 17:59:34.585 +08:00] [INFO] [main.go:234] ["0 delete success: 700"] [2024/05/31 17:59:34.705 +08:00] [INFO] [main.go:220] ["1 insert success: 1700"] table savepoint.finish_mark exists check diff successfully + set +x + tso='450140457771991041 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140457771991041 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 17:59:34 CST 2024] <<<<<< START cdc server in generate_column case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.3451134513.out server --log-file /tmp/tidb_cdc_test/generate_column/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/generate_column/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/31 17:59:34.925 +08:00] [INFO] [main.go:220] ["0 insert success: 1500"] [2024/05/31 17:59:35.016 +08:00] [INFO] [main.go:220] ["1 insert success: 1800"] [2024/05/31 17:59:35.020 +08:00] [INFO] [main.go:234] ["1 delete success: 900"] [2024/05/31 17:59:35.289 +08:00] [INFO] [main.go:220] ["0 insert success: 1600"] [2024/05/31 17:59:35.294 +08:00] [INFO] [main.go:234] ["0 delete success: 800"] wait process cdc.test exit for 1-th time... [2024/05/31 17:59:35.355 +08:00] [INFO] [main.go:220] ["1 insert success: 1900"] wait process cdc.test exit for 2-th time... [2024/05/31 17:59:35.727 +08:00] [INFO] [main.go:220] ["0 insert success: 1700"] [2024/05/31 17:59:35.760 +08:00] [INFO] [main.go:220] ["1 insert success: 2000"] [2024/05/31 17:59:35.764 +08:00] [INFO] [main.go:234] ["1 delete success: 1000"] ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns not exists for 9-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:59:36 CST 2024] <<<<<< run test case savepoint success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:59:37 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa509502-d395-49a5-9a94-ccbf73f4915d {"id":"fa509502-d395-49a5-9a94-ccbf73f4915d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149575} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16f71f85 fa509502-d395-49a5-9a94-ccbf73f4915d /tidb/cdc/default/default/upstream/7375101224964960690 {"id":7375101224964960690,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa509502-d395-49a5-9a94-ccbf73f4915d {"id":"fa509502-d395-49a5-9a94-ccbf73f4915d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149575} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16f71f85 fa509502-d395-49a5-9a94-ccbf73f4915d /tidb/cdc/default/default/upstream/7375101224964960690 {"id":7375101224964960690,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa509502-d395-49a5-9a94-ccbf73f4915d {"id":"fa509502-d395-49a5-9a94-ccbf73f4915d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149575} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce16f71f85 fa509502-d395-49a5-9a94-ccbf73f4915d /tidb/cdc/default/default/upstream/7375101224964960690 {"id":7375101224964960690,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.generate_column.cli.34556.out cli changefeed create --start-ts=450140457771991041 '--sink-uri=kafka://127.0.0.1:9092/ticdc-generate-column-test-19130?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' table test.binary_columns exists check diff failed 1-th time, retry later Create changefeed successfully! ID: 0f6ef1c6-a740-41d3-8f10-c5b4efdad5d9 Info: {"upstream_id":7375101224964960690,"namespace":"default","id":"0f6ef1c6-a740-41d3-8f10-c5b4efdad5d9","sink_uri":"kafka://127.0.0.1:9092/ticdc-generate-column-test-19130?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T17:59:38.391681038+08:00","start_ts":450140457771991041,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140457771991041,"checkpoint_ts":450140457771991041,"checkpoint_time":"2024-05-31 17:59:33.410"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x [Fri May 31 17:59:39 CST 2024] <<<<<< START kafka consumer in generate_column case >>>>>> ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 7037 0 --:--:-- --:--:-- --:--:-- 7129 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 17:59:26.278","puller_resolved_ts":"2024-05-31 17:59:19.279","last_synced_ts":"2024-05-31 17:57:10.879","now_ts":"2024-05-31 17:59:28.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '17:59:26.278","puller_resolved_ts":"2024-05-31' '17:59:19.279","last_synced_ts":"2024-05-31' '17:57:10.879","now_ts":"2024-05-31' '17:59:28.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9634 8.6 0.0 13586636 144804 ? Sl 17:56 0:13 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9692 5.8 0.0 13374596 137520 ? Sl 17:56 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2024-05-31T17:59:34.176387+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T17:59:34.176456+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T17:59:34.182752+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T17:59:34.182813+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T17:59:34.185515+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:34.189892+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:34.243714+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-31T17:59:34.243781+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T17:59:34.251775+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1717149576.0100286,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc004f18540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717149576.0101042,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717149576.033447,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002820e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717149576.0334878,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717149576.05054,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00538a380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717149576.0506008,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table generate_column.t not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) {"level":"warn","ts":"2024-05-31T17:59:40.186696+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:40.191153+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:40.252803+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... [2024/05/31 17:59:41.153 +08:00] [WARN] [diff.go:180] ["table struct is not equal"] [reason="column num not equal, one is 5 another is 4"] table generate_column.t not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.binary_columns exists check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... [2024/05/31 17:59:43.361 +08:00] [INFO] [main.go:78] ["runDDLTest take 1m4.859875726s"] table mark.finish_mark_0 exists table mark.finish_mark_1 exists table mark.finish_mark_2 exists table mark.finish_mark_3 exists table mark.finish_mark_4 exists table mark.finish_mark_5 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table generate_column.t exists table generate_column.t1 exists check diff failed 1-th time, retry later check diff failed 2-th time, retry later table mark.finish_mark_5 exists table mark.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... {"level":"warn","ts":"2024-05-31T17:59:46.187984+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:46.19225+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:46.25425+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later table mark.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 17:59:47 CST 2024] <<<<<< run test case generate_column success! >>>>>> ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: kafka... <<================= wait process cdc.test exit for 2-th time... +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri May 31 17:59:49 CST 2024] <<<<<< run test case multi_source success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385d80d40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:22482, start at 2024-05-31 17:59:49.59271345 +0800 CST m=+1.790158063 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:49.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:49.557 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:49.557 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385d80d40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:22482, start at 2024-05-31 17:59:49.59271345 +0800 CST m=+1.790158063 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:49.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:49.557 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:49.557 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385d80c40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:22553, start at 2024-05-31 17:59:49.581488481 +0800 CST m=+1.706848506 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:49.587 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:49.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:49.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... check diff failed 5-th time, retry later Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.23861.out cli tso query --pd=http://127.0.0.1:2379 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:59:52.189322+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:52.193898+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:52.255904+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} check diff failed 6-th time, retry later [2024/05/31 17:59:51.694 +08:00] [WARN] [diff.go:557] ["checksum is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [sourceChecksum=0] [targetChecksum=1828936557] [getSourceChecksumCost=1.80451ms] [getTargetChecksumCost=1.226581ms] + set +x + tso='450140462751940609 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140462751940609 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Fri May 31 17:59:53 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2389923901.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 7-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 8-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 09:59:56 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fe1835f-c541-43e8-b63f-4091826e02fe {"id":"7fe1835f-c541-43e8-b63f-4091826e02fe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149594} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1747658e 7fe1835f-c541-43e8-b63f-4091826e02fe /tidb/cdc/default/default/upstream/7375101307982948528 {"id":7375101307982948528,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fe1835f-c541-43e8-b63f-4091826e02fe {"id":"7fe1835f-c541-43e8-b63f-4091826e02fe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149594} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1747658e 7fe1835f-c541-43e8-b63f-4091826e02fe /tidb/cdc/default/default/upstream/7375101307982948528 {"id":7375101307982948528,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7fe1835f-c541-43e8-b63f-4091826e02fe {"id":"7fe1835f-c541-43e8-b63f-4091826e02fe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149594} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1747658e 7fe1835f-c541-43e8-b63f-4091826e02fe /tidb/cdc/default/default/upstream/7375101307982948528 {"id":7375101307982948528,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-31T17:59:58.190694+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:58.195228+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T17:59:58.257752+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385e08280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:17006, start at 2024-05-31 17:59:58.219689798 +0800 CST m=+1.804109088 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:58.226 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:58.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:58.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385e08280003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:17006, start at 2024-05-31 17:59:58.219689798 +0800 CST m=+1.804109088 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:58.226 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:58.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:58.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385e067c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:17079, start at 2024-05-31 17:59:58.147984027 +0800 CST m=+1.676240809 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:01:58.154 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:59:58.111 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:49:58.111 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 9-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18397.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 10-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } + set +x + tso='450140465009262593 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140465009262593 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140465009262593 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 18:00:02 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1842918431.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // withEnv ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff successfully wait process cdc.test exit for 1-th time... 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:00:04.19162+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:00:04.196201+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:00:04.259053+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:00:04 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:00:05 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/286d4cfa-42f4-4e05-9216-35eac82713ac {"id":"286d4cfa-42f4-4e05-9216-35eac82713ac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce17670f85 286d4cfa-42f4-4e05-9216-35eac82713ac /tidb/cdc/default/default/upstream/7375101347036512920 {"id":7375101347036512920,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/286d4cfa-42f4-4e05-9216-35eac82713ac {"id":"286d4cfa-42f4-4e05-9216-35eac82713ac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce17670f85 286d4cfa-42f4-4e05-9216-35eac82713ac /tidb/cdc/default/default/upstream/7375101347036512920 {"id":7375101347036512920,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/286d4cfa-42f4-4e05-9216-35eac82713ac {"id":"286d4cfa-42f4-4e05-9216-35eac82713ac","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149602} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce17670f85 286d4cfa-42f4-4e05-9216-35eac82713ac /tidb/cdc/default/default/upstream/7375101347036512920 {"id":7375101347036512920,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140465009262593 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.18484.out cli changefeed create --start-ts=450140465009262593 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375101347036512920,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T18:00:05.965987289+08:00","start_ts":450140465009262593,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140465009262593,"checkpoint_ts":450140465009262593,"checkpoint_time":"2024-05-31 18:00:01.018"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 14852 0 --:--:-- --:--:-- --:--:-- 15785 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 18:00:01.018","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-31 18:00:07.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '18:00:01.018","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '18:00:07.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '18:00:01.018","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '18:00:07.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-31 18:00:01.018' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '18:00:01.018","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '18:00:07.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '18:00:01.018","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '18:00:07.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-31 18:00:07' + echo 'sink_checkpoint_ts is 2024-05-31' 18:00:01.018 sink_checkpoint_ts is 2024-05-31 18:00:01.018 ++ date -d '2024-05-31 18:00:01.018' +%s + checkpoint_timestamp=1717149601 ++ date -d '2024-05-31 18:00:07' +%s + current_timestamp=1717149607 + '[' 6 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:00:09.178508+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T18:00:09.178556+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T18:00:09.183738+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T18:00:09.183788+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T18:00:09.24494+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-31T18:00:09.244983+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table test.t1 exists + sleep 5 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:00:10.192878+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:00:10.196802+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:00:10.259828+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) {"level":"warn","ts":1717149611.0110161,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc004f18540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717149611.0110695,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717149611.0348809,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002820e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717149611.034946,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717149611.05193,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00538a380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717149611.0519748,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 4432 0 --:--:-- --:--:-- --:--:-- 4500 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 18:00:14.468","puller_resolved_ts":"2024-05-31 18:00:07.568","last_synced_ts":"2024-05-31 18:00:08.068","now_ts":"2024-05-31 18:00:15.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:00:14.468","puller_resolved_ts":"2024-05-31' '18:00:07.568","last_synced_ts":"2024-05-31' '18:00:08.068","now_ts":"2024-05-31' '18:00:15.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:00:14.468","puller_resolved_ts":"2024-05-31' '18:00:07.568","last_synced_ts":"2024-05-31' '18:00:08.068","now_ts":"2024-05-31' '18:00:15.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:00:16.193857+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001b481c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:00:16.197484+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0016396c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:00:16.26162+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001770380/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test start tidb cluster in /tmp/tidb_cdc_test/canal_json_basic Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Fri May 31 18:00:16 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ true != \n\o ]] + set +x + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.2398023982.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385fb0d00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:25781, start at 2024-05-31 18:00:25.402852508 +0800 CST m=+1.809147547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:25.409 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:00:25.396 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:50:25.396 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385fb0d00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:25781, start at 2024-05-31 18:00:25.402852508 +0800 CST m=+1.809147547 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:25.409 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:00:25.396 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:50:25.396 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385fb1700014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:25863, start at 2024-05-31 18:00:25.458910563 +0800 CST m=+1.807280315 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:25.465 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:00:25.436 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:50:25.436 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_basic/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [Fri May 31 18:00:27 CST 2024] <<<<<< START cdc server in canal_json_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.2718027182.out server --log-file /tmp/tidb_cdc_test/canal_json_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:00:30 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8494cc82-5bb3-4b8b-b2cd-05b80c7487a5 {"id":"8494cc82-5bb3-4b8b-b2cd-05b80c7487a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149628} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce17cf457d 8494cc82-5bb3-4b8b-b2cd-05b80c7487a5 /tidb/cdc/default/default/upstream/7375101463923398039 {"id":7375101463923398039,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8494cc82-5bb3-4b8b-b2cd-05b80c7487a5 {"id":"8494cc82-5bb3-4b8b-b2cd-05b80c7487a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149628} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce17cf457d 8494cc82-5bb3-4b8b-b2cd-05b80c7487a5 /tidb/cdc/default/default/upstream/7375101463923398039 {"id":7375101463923398039,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8494cc82-5bb3-4b8b-b2cd-05b80c7487a5 {"id":"8494cc82-5bb3-4b8b-b2cd-05b80c7487a5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149628} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce17cf457d 8494cc82-5bb3-4b8b-b2cd-05b80c7487a5 /tidb/cdc/default/default/upstream/7375101463923398039 {"id":7375101463923398039,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_basic.cli.27222.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-canal-json-basic?protocol=canal-json&enable-tidb-extension=true' chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Create changefeed successfully! ID: feb31c29-cfc7-464d-b1f0-bea83be35172 Info: {"upstream_id":7375101463923398039,"namespace":"default","id":"feb31c29-cfc7-464d-b1f0-bea83be35172","sink_uri":"kafka://127.0.0.1:9092/ticdc-canal-json-basic?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-31T18:00:31.433189253+08:00","start_ts":450140472946458628,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140472946458628,"checkpoint_ts":450140472946458628,"checkpoint_time":"2024-05-31 18:00:31.296"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x 6:00PM INF > Run case=sql/debezium/binary_column_test.sql 6:00PM INF > Run case=sql/debezium/binary_mode_test.sql ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Fri May 31 18:00:37 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 6:00PM INF > Run case=sql/debezium/connector_test.sql [Fri May 31 18:00:37 CST 2024] <<<<<< START kafka consumer in canal_json_basic case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38607d08000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:13196, start at 2024-05-31 18:00:38.475759986 +0800 CST m=+1.714819700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:38.482 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:00:38.466 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:50:38.466 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38607d08000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:13196, start at 2024-05-31 18:00:38.475759986 +0800 CST m=+1.714819700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:38.482 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:00:38.466 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:50:38.466 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f38607f480015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:13273, start at 2024-05-31 18:00:38.656370928 +0800 CST m=+1.837836812 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:38.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:00:38.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:50:38.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14691.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='450140475586510849 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140475586510849 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140475586510849 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri May 31 18:00:42 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1472214724.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table test.finish_mark not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:00:45 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be841118-0299-41c3-add1-3e99d8991b8a {"id":"be841118-0299-41c3-add1-3e99d8991b8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149643} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1800c588 be841118-0299-41c3-add1-3e99d8991b8a /tidb/cdc/default/default/upstream/7375101517537124401 {"id":7375101517537124401,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be841118-0299-41c3-add1-3e99d8991b8a {"id":"be841118-0299-41c3-add1-3e99d8991b8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149643} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1800c588 be841118-0299-41c3-add1-3e99d8991b8a /tidb/cdc/default/default/upstream/7375101517537124401 {"id":7375101517537124401,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/be841118-0299-41c3-add1-3e99d8991b8a {"id":"be841118-0299-41c3-add1-3e99d8991b8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149643} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1800c588 be841118-0299-41c3-add1-3e99d8991b8a /tidb/cdc/default/default/upstream/7375101517537124401 {"id":7375101517537124401,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140475586510849 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14782.out cli changefeed create --start-ts=450140475586510849 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375101517537124401,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T18:00:46.341805079+08:00","start_ts":450140475586510849,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140475586510849,"checkpoint_ts":450140475586510849,"checkpoint_time":"2024-05-31 18:00:41.367"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 4-th check, retry later + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... table test.t1 exists + sleep 5 table test.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 12551 39.7 0.5 4729056 2283776 ? Sl 18:00 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 12552 29.4 0.5 4690148 2216912 ? Sl 18:00 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 12553 29.4 0.5 4692196 2215744 ? Sl 18:00 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 12555 38.7 0.5 4722400 2275560 ? Sl 18:00 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 8914 0 --:--:-- --:--:-- --:--:-- 9000 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 18:00:51.816","puller_resolved_ts":"2024-05-31 18:00:47.766","last_synced_ts":"2024-05-31 18:00:47.766","now_ts":"2024-05-31 18:00:54.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:00:51.816","puller_resolved_ts":"2024-05-31' '18:00:47.766","last_synced_ts":"2024-05-31' '18:00:47.766","now_ts":"2024-05-31' '18:00:54.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:00:51.816","puller_resolved_ts":"2024-05-31' '18:00:47.766","last_synced_ts":"2024-05-31' '18:00:47.766","now_ts":"2024-05-31' '18:00:54.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3861ad200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:25094, start at 2024-05-31 18:00:57.973300152 +0800 CST m=+1.629724486 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:57.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:00:57.978 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:50:57.978 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3861ad200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:25094, start at 2024-05-31 18:00:57.973300152 +0800 CST m=+1.629724486 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:57.979 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:00:57.978 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:50:57.978 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3861b0dc0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:25163, start at 2024-05-31 18:00:58.211377406 +0800 CST m=+1.817223903 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:58.218 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:00:58.218 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:50:58.218 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 11-th check, retry later [Fri May 31 18:01:00 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.2646126463.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 12-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Failed to start upstream TiDB start tidb cluster failed The 2 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:01:03 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2e969e00-9277-49b0-8a3b-95b9370f5cf7 {"id":"2e969e00-9277-49b0-8a3b-95b9370f5cf7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149660} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18530b80 2e969e00-9277-49b0-8a3b-95b9370f5cf7 /tidb/cdc/default/default/upstream/7375101600291347993 {"id":7375101600291347993,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2e969e00-9277-49b0-8a3b-95b9370f5cf7 {"id":"2e969e00-9277-49b0-8a3b-95b9370f5cf7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149660} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18530b80 2e969e00-9277-49b0-8a3b-95b9370f5cf7 /tidb/cdc/default/default/upstream/7375101600291347993 {"id":7375101600291347993,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2e969e00-9277-49b0-8a3b-95b9370f5cf7 {"id":"2e969e00-9277-49b0-8a3b-95b9370f5cf7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149660} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18530b80 2e969e00-9277-49b0-8a3b-95b9370f5cf7 /tidb/cdc/default/default/upstream/7375101600291347993 {"id":7375101600291347993,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.26503.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-event-filter-20674?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml Create changefeed successfully! ID: 338ddeec-c0e3-4049-88c5-4961d3e26b71 Info: {"upstream_id":7375101600291347993,"namespace":"default","id":"338ddeec-c0e3-4049-88c5-4961d3e26b71","sink_uri":"kafka://127.0.0.1:9092/ticdc-event-filter-20674?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T18:01:04.100455237+08:00","start_ts":450140481513848840,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140481513848840,"checkpoint_ts":450140481513848840,"checkpoint_time":"2024-05-31 18:01:03.978"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 13-th check, retry later + set +x [Fri May 31 18:01:05 CST 2024] <<<<<< START kafka consumer in event_filter case >>>>>> table event_filter.t1 does not exists table event_filter.t1 not exists for 1-th check, retry later table test.finish_mark not exists for 14-th check, retry later table event_filter.t1 not exists for 2-th check, retry later table test.finish_mark not exists for 15-th check, retry later table event_filter.t1 exists table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later table test.finish_mark not exists for 16-th check, retry later table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later table test.finish_mark not exists for 17-th check, retry later table event_filter.t_alter exists table event_filter.finish_mark exists check diff failed 1-th time, retry later table test.finish_mark exists check diff successfully check diff failed 2-th time, retry later 6:01PM INF > Run case=sql/debezium/connector_test_ro.sql table test.finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... check diff successfully wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:01:20 CST 2024] <<<<<< run test case event_filter success! >>>>>> table test.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 6-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385a68340013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:10889, start at 2024-05-31 17:58:58.855980464 +0800 CST m=+2.050551551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:58.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:58.829 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:58.829 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385a68340013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:10889, start at 2024-05-31 17:58:58.855980464 +0800 CST m=+2.050551551 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:58.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:58.829 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:58.829 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f385a69080014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-p2gjp-nlslk, pid:10963, start at 2024-05-31 17:58:58.907871872 +0800 CST m=+2.041285250 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:02:58.941 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-17:58:58.882 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:48:58.882 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.finish_mark not exists for 7-th check, retry later [2024/05/31 18:01:26.248 +08:00] [WARN] [diff.go:766] ["rows is not equal"] [table=`test`.`ntest`] [where="((TRUE) AND TRUE)"] [cost=1m34.554511157s] [2024/05/31 18:01:26.251 +08:00] [WARN] [diff.go:384] ["check chunk data not equal"] [chunk="{\"id\":0,\"bounds\":[],\"where\":\"((TRUE) AND TRUE)\",\"args\":null,\"state\":\"failed\"}"] + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.13886.out cli tso query --pd=http://127.0.0.1:2379 table test.finish_mark not exists for 8-th check, retry later [2024/05/31 18:01:31.842 +08:00] [INFO] [dailytest.go:68] ["test pass!!!"] =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: kafka... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=kafka + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... + set +x + tso='450140488639447044 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140488639447044 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 18:01:32 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1392013922.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... table test.finish_mark not exists for 9-th check, retry later wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:01:33 CST 2024] <<<<<< run test case cdc success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... table test.finish_mark exists check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:01:35 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f408cc4-b990-4138-becd-aa9c4637b6fa {"id":"6f408cc4-b990-4138-becd-aa9c4637b6fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149692} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18b8e8df 6f408cc4-b990-4138-becd-aa9c4637b6fa /tidb/cdc/default/default/upstream/7375101076099853747 {"id":7375101076099853747,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f408cc4-b990-4138-becd-aa9c4637b6fa {"id":"6f408cc4-b990-4138-becd-aa9c4637b6fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149692} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18b8e8df 6f408cc4-b990-4138-becd-aa9c4637b6fa /tidb/cdc/default/default/upstream/7375101076099853747 {"id":7375101076099853747,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6f408cc4-b990-4138-becd-aa9c4637b6fa {"id":"6f408cc4-b990-4138-becd-aa9c4637b6fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149692} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18b8e8df 6f408cc4-b990-4138-becd-aa9c4637b6fa /tidb/cdc/default/default/upstream/7375101076099853747 {"id":7375101076099853747,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.13975.out cli changefeed create --start-ts=450140488639447044 '--sink-uri=kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-25680?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' wait process cdc.test exit for 1-th time... Create changefeed successfully! ID: 1dc947e2-3e47-425f-ba8e-f3f69c8a3bc4 Info: {"upstream_id":7375101076099853747,"namespace":"default","id":"1dc947e2-3e47-425f-ba8e-f3f69c8a3bc4","sink_uri":"kafka://127.0.0.1:9092/ticdc-resourcecontrol-test-25680?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T18:01:36.161134108+08:00","start_ts":450140488639447044,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140488639447044,"checkpoint_ts":450140488639447044,"checkpoint_time":"2024-05-31 18:01:31.160"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Fri May 31 18:01:37 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> + set +x [Fri May 31 18:01:37 CST 2024] <<<<<< START kafka consumer in resourcecontrol case >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27788, start at 2024-05-31 18:01:41.7025106 +0800 CST m=+1.772085184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27788, start at 2024-05-31 18:01:41.7025106 +0800 CST m=+1.772085184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458d4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27856, start at 2024-05-31 18:01:41.701685492 +0800 CST m=+1.718288255 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.708 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.29284.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + set +x + tso='450140492159516673 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140492159516673 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140492159516673 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Fri May 31 18:01:45 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.2931629318.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:01:49 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef221c21-ceb2-42e5-b5ca-bf59f85df030 {"id":"ef221c21-ceb2-42e5-b5ca-bf59f85df030","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149706} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18fb7289 ef221c21-ceb2-42e5-b5ca-bf59f85df030 /tidb/cdc/default/default/upstream/7375101789149404255 {"id":7375101789149404255,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef221c21-ceb2-42e5-b5ca-bf59f85df030 {"id":"ef221c21-ceb2-42e5-b5ca-bf59f85df030","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149706} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18fb7289 ef221c21-ceb2-42e5-b5ca-bf59f85df030 /tidb/cdc/default/default/upstream/7375101789149404255 {"id":7375101789149404255,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ef221c21-ceb2-42e5-b5ca-bf59f85df030 {"id":"ef221c21-ceb2-42e5-b5ca-bf59f85df030","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149706} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18fb7289 ef221c21-ceb2-42e5-b5ca-bf59f85df030 /tidb/cdc/default/default/upstream/7375101789149404255 {"id":7375101789149404255,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140492159516673 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.29366.out cli changefeed create --start-ts=450140492159516673 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375101789149404255,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T18:01:49.441104603+08:00","start_ts":450140492159516673,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140492159516673,"checkpoint_ts":450140492159516673,"checkpoint_time":"2024-05-31 18:01:44.588"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 + '[' kafka == mysql ']' + stop_tidb_cluster start tidb cluster in /tmp/tidb_cdc_test/canal_json_content_compatible Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 6:01PM INF > Run case=sql/debezium/datetime_key_test.sql Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 6:01PM INF > Run case=sql/debezium/db_default_charset.sql Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386575ec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2, pid:24657, start at 2024-05-31 18:01:59.95867719 +0800 CST m=+1.835281937 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:59.967 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:59.931 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:59.931 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386575ec0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2, pid:24657, start at 2024-05-31 18:01:59.95867719 +0800 CST m=+1.835281937 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:59.967 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:59.931 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:59.931 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386573840015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lchjd-g05k2, pid:24741, start at 2024-05-31 18:01:59.824889592 +0800 CST m=+1.646446319 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:59.831 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:59.827 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:59.827 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386564500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:28560, start at 2024-05-31 18:01:58.834011116 +0800 CST m=+1.887231946 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:58.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:58.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:58.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386564500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:28560, start at 2024-05-31 18:01:58.834011116 +0800 CST m=+1.887231946 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:58.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:58.804 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:58.804 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386564ec0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:28633, start at 2024-05-31 18:01:58.873020424 +0800 CST m=+1.872218129 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:58.882 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:58.843 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:58.843 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_content_compatible/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.26189.out cli tso query --pd=http://127.0.0.1:2379 6:02PM INF > Run case=sql/debezium/db_default_charset_noutf.sql [Fri May 31 18:02:03 CST 2024] <<<<<< START cdc server in canal_json_content_compatible case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_content_compatible.3008130083.out server --log-file /tmp/tidb_cdc_test/canal_json_content_compatible/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_content_compatible/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... + set +x + tso='450140496915857409 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140496915857409 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 18:02:04 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.2623226234.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:02:06 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0e499e8-459a-4c24-9c66-166fd62f96a6 {"id":"e0e499e8-459a-4c24-9c66-166fd62f96a6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149723} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce193fdb83 e0e499e8-459a-4c24-9c66-166fd62f96a6 /tidb/cdc/default/default/upstream/7375101863055247948 {"id":7375101863055247948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0e499e8-459a-4c24-9c66-166fd62f96a6 {"id":"e0e499e8-459a-4c24-9c66-166fd62f96a6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149723} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce193fdb83 e0e499e8-459a-4c24-9c66-166fd62f96a6 /tidb/cdc/default/default/upstream/7375101863055247948 {"id":7375101863055247948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e0e499e8-459a-4c24-9c66-166fd62f96a6 {"id":"e0e499e8-459a-4c24-9c66-166fd62f96a6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149723} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce193fdb83 e0e499e8-459a-4c24-9c66-166fd62f96a6 /tidb/cdc/default/default/upstream/7375101863055247948 {"id":7375101863055247948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_content_compatible.cli.30131.out cli changefeed create '--sink-uri=kafka://127.0.0.1:9092/ticdc-canal-json-content-compatible?protocol=canal-json&enable-tidb-extension=true&content-compatible=true' start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Create changefeed successfully! ID: 7da5bed0-910e-4e5f-a7ef-3fd0c9ddc638 Info: {"upstream_id":7375101863055247948,"namespace":"default","id":"7da5bed0-910e-4e5f-a7ef-3fd0c9ddc638","sink_uri":"kafka://127.0.0.1:9092/ticdc-canal-json-content-compatible?protocol=canal-json\u0026enable-tidb-extension=true\u0026content-compatible=true","create_time":"2024-05-31T18:02:06.790554041+08:00","start_ts":450140497943986177,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":true,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140497943986177,"checkpoint_ts":450140497943986177,"checkpoint_time":"2024-05-31 18:02:06.654"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:02:07 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e965225-06f1-45a3-9894-697b4ae75816 {"id":"1e965225-06f1-45a3-9894-697b4ae75816","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149724} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce194283a9 1e965225-06f1-45a3-9894-697b4ae75816 /tidb/cdc/default/default/upstream/7375101871071472917 {"id":7375101871071472917,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e965225-06f1-45a3-9894-697b4ae75816 {"id":"1e965225-06f1-45a3-9894-697b4ae75816","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149724} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce194283a9 1e965225-06f1-45a3-9894-697b4ae75816 /tidb/cdc/default/default/upstream/7375101871071472917 {"id":7375101871071472917,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1e965225-06f1-45a3-9894-697b4ae75816 {"id":"1e965225-06f1-45a3-9894-697b4ae75816","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149724} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce194283a9 1e965225-06f1-45a3-9894-697b4ae75816 /tidb/cdc/default/default/upstream/7375101871071472917 {"id":7375101871071472917,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.26290.out cli changefeed create --start-ts=450140496915857409 '--sink-uri=kafka://127.0.0.1:9092/ticdc-split-region-test-30943?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml Create changefeed successfully! ID: ec5bb352-6030-4e4d-9d57-7c20d7f2f052 Info: {"upstream_id":7375101871071472917,"namespace":"default","id":"ec5bb352-6030-4e4d-9d57-7c20d7f2f052","sink_uri":"kafka://127.0.0.1:9092/ticdc-split-region-test-30943?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T18:02:07.730071803+08:00","start_ts":450140496915857409,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140496915857409,"checkpoint_ts":450140496915857409,"checkpoint_time":"2024-05-31 18:02:02.732"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... 6:02PM INF > Run case=sql/debezium/decimal_column_test.sql + set +x + set +x [Fri May 31 18:02:09 CST 2024] <<<<<< START kafka consumer in split_region case >>>>>> table split_region.test1 not exists for 1-th check, retry later table split_region.test1 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 6:02PM INF > Run case=sql/debezium/enum_column_test.sql table split_region.test1 exists table split_region.test2 exists check diff failed 1-th time, retry later [Fri May 31 18:02:13 CST 2024] <<<<<< START kafka consumer in canal_json_content_compatible case >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27788, start at 2024-05-31 18:01:41.7025106 +0800 CST m=+1.772085184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27788, start at 2024-05-31 18:01:41.7025106 +0800 CST m=+1.772085184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458d4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27856, start at 2024-05-31 18:01:41.701685492 +0800 CST m=+1.718288255 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.708 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... 6:02PM INF > Run case=sql/debezium/multitable_dbz_871.sql start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... 6:02PM INF > Run case=sql/debezium/mysql_dbz_6533.sql check diff failed 1-th time, retry later Verifying downstream PD is started... check diff failed 2-th time, retry later table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:02:33 CST 2024] <<<<<< run test case split_region success! >>>>>> table resourcecontrol.finish_mark not exists for 1-th check, retry later table resourcecontrol.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:02:30 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> table test.finish_mark not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27788, start at 2024-05-31 18:01:41.7025106 +0800 CST m=+1.772085184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27788, start at 2024-05-31 18:01:41.7025106 +0800 CST m=+1.772085184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458d4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27856, start at 2024-05-31 18:01:41.701685492 +0800 CST m=+1.718288255 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.708 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 4010 0 --:--:-- --:--:-- --:--:-- 3946 100 221 100 221 0 0 4002 0 --:--:-- --:--:-- --:--:-- 3946 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 18:02:24.618","puller_resolved_ts":"2024-05-31 18:02:16.568","last_synced_ts":"2024-05-31 18:00:08.068","now_ts":"2024-05-31 18:02:25.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '18:02:24.618","puller_resolved_ts":"2024-05-31' '18:02:16.568","last_synced_ts":"2024-05-31' '18:00:08.068","now_ts":"2024-05-31' '18:02:25.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 16125 8.2 0.0 14194680 144688 ? Sl 17:59 0:12 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 16188 5.7 0.0 14129848 139404 ? Sl 17:59 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":"2024-05-31T18:02:31.465944+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T18:02:31.465995+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T18:02:31.466747+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T18:02:31.466815+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T18:02:31.47706+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:31.477872+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:31.526179+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-31T18:02:31.52624+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T18:02:31.535307+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1717149753.185674,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0034eae00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717149753.185716,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717149753.2550588,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003236380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717149753.2551286,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717149753.4261305,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017cfdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717149753.426179,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T18:02:37.478528+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:37.47937+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:37.535978+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark not exists for 5-th check, retry later table test.finish_mark not exists for 6-th check, retry later table test.finish_mark not exists for 7-th check, retry later 6:02PM INF > Run case=sql/debezium/nationalized_character_test.sql \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } {"level":"warn","ts":"2024-05-31T18:02:43.480493+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:43.48053+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:43.537437+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [Pipeline] // timeout table test.finish_mark not exists for 8-th check, retry later [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } table test.finish_mark not exists for 9-th check, retry later [Pipeline] // stage [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... 6:02PM INF > Run case=sql/debezium/numeric_column_test.sql table test.finish_mark not exists for 10-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:02:49.481657+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:49.48169+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:49.538175+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark not exists for 11-th check, retry later table test.finish_mark exists Verifying downstream PD is started... check diff successfully Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 6:02PM INF > Run case=sql/debezium/readbinlog_test.sql Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... 6:02PM INF > Run case=sql/debezium/real_test.sql Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:02:55.482456+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:55.482705+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:02:55.538959+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27788, start at 2024-05-31 18:01:41.7025106 +0800 CST m=+1.772085184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458e00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27788, start at 2024-05-31 18:01:41.7025106 +0800 CST m=+1.772085184 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.688 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.688 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386458d4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-wlvl3-gs0cd, pid:27856, start at 2024-05-31 18:01:41.701685492 +0800 CST m=+1.718288255 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:03:41.708 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:01:41.685 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:51:41.685 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.33380.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140511140052995 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140511140052995 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140511140052995 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Fri May 31 18:02:58 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.3341533417.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:03:01.483567+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:03:01.48405+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:03:01.540675+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:03:01 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7375101789149404255 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-31 18:01:49.441104603 +0800 CST StartTs:450140492159516673 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003ad6240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140493431177217} {CheckpointTs:450140493797916705 MinTableBarrierTs:450140493837238273 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec88bd52-a56a-4d9f-8ad6-1f4724d906e2 {"id":"ec88bd52-a56a-4d9f-8ad6-1f4724d906e2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149778} /tidb/cdc/default/__cdc_meta__/capture/ef221c21-ceb2-42e5-b5ca-bf59f85df030 {"id":"ef221c21-ceb2-42e5-b5ca-bf59f85df030","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149706} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18fb7289 ef221c21-ceb2-42e5-b5ca-bf59f85df030 /tidb/cdc/default/__cdc_meta__/owner/22318fce1a10cbd3 ec88bd52-a56a-4d9f-8ad6-1f4724d906e2 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7375101789149404255,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T18:01:49.441104603+08:00","start-ts":450140492159516673,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140493431177217} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":450140493797916705,"min-table-barrier-ts":450140493837238273,"admin-job-type":0} /tidb/cdc/default/default/task/position/ec88bd52-a56a-4d9f-8ad6-1f4724d906e2/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ef221c21-ceb2-42e5-b5ca-bf59f85df030/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375101789149404255 {"id":7375101789149404255,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7375101789149404255 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-31 18:01:49.441104603 +0800 CST StartTs:450140492159516673 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003ad6240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140493431177217} {CheckpointTs:450140493797916705 MinTableBarrierTs:450140493837238273 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec88bd52-a56a-4d9f-8ad6-1f4724d906e2 {"id":"ec88bd52-a56a-4d9f-8ad6-1f4724d906e2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149778} /tidb/cdc/default/__cdc_meta__/capture/ef221c21-ceb2-42e5-b5ca-bf59f85df030 {"id":"ef221c21-ceb2-42e5-b5ca-bf59f85df030","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149706} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18fb7289 ef221c21-ceb2-42e5-b5ca-bf59f85df030 /tidb/cdc/default/__cdc_meta__/owner/22318fce1a10cbd3 ec88bd52-a56a-4d9f-8ad6-1f4724d906e2 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7375101789149404255,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T18:01:49.441104603+08:00","start-ts":450140492159516673,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140493431177217} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":450140493797916705,"min-table-barrier-ts":450140493837238273,"admin-job-type":0} /tidb/cdc/default/default/task/position/ec88bd52-a56a-4d9f-8ad6-1f4724d906e2/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ef221c21-ceb2-42e5-b5ca-bf59f85df030/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null+ grep -q 'failed to get info:' } /tidb/cdc/default/default/upstream/7375101789149404255 {"id":7375101789149404255,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7375101789149404255 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-31 18:01:49.441104603 +0800 CST StartTs:450140492159516673 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003ad6240 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-75-g18cf66054 Epoch:450140493431177217} {CheckpointTs:450140493797916705 MinTableBarrierTs:450140493837238273 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ec88bd52-a56a-4d9f-8ad6-1f4724d906e2 {"id":"ec88bd52-a56a-4d9f-8ad6-1f4724d906e2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149778} /tidb/cdc/default/__cdc_meta__/capture/ef221c21-ceb2-42e5-b5ca-bf59f85df030 {"id":"ef221c21-ceb2-42e5-b5ca-bf59f85df030","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149706} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce18fb7289 ef221c21-ceb2-42e5-b5ca-bf59f85df030 /tidb/cdc/default/__cdc_meta__/owner/22318fce1a10cbd3 ec88bd52-a56a-4d9f-8ad6-1f4724d906e2 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7375101789149404255,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-31T18:01:49.441104603+08:00","start-ts":450140492159516673,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-75-g18cf66054","epoch":450140493431177217} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":450140493797916705,"min-table-barrier-ts":450140493837238273,"admin-job-type":0} /tidb/cdc/default/default/task/position/ec88bd52-a56a-4d9f-8ad6-1f4724d906e2/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ef221c21-ceb2-42e5-b5ca-bf59f85df030/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7375101789149404255 {"id":7375101789149404255,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140511140052995 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.33471.out cli changefeed create --start-ts=450140511140052995 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 Create changefeed successfully! ID: test-2 Info: {"upstream_id":7375101789149404255,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T18:03:03.448095456+08:00","start_ts":450140511140052995,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140511140052995,"checkpoint_ts":450140511140052995,"checkpoint_time":"2024-05-31 18:02:56.993"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table test.finish_mark not exists for 1-th check, retry later 6:03PM INF > Run case=sql/debezium/regression_test.sql table test.finish_mark not exists for 2-th check, retry later + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:03:06.467068+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T18:03:06.467106+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T18:03:06.468325+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-31T18:03:06.468374+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-31T18:03:06.527495+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-31T18:03:06.527535+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:03:07.484578+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:03:07.484603+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:03:07.541439+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table test.finish_mark not exists for 3-th check, retry later {"level":"warn","ts":1717149788.186747,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0034eae00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1717149788.186799,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1717149788.2558944,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003236380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717149788.2559352,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0{"level":"warn","ts":1717149788.4280446,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017cfdc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1717149788.4280927,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table test.finish_mark not exists for 4-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:03:12 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0{"level":"warn","ts":"2024-05-31T18:03:13.485699+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001641c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:03:13.486341+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0017ce000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-31T18:03:13.542838+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000948000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' ++ jq -r .error_code + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 25129 0 --:--:-- --:--:-- --:--:-- 25821 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 18:00:54.816","puller_resolved_ts":"2024-05-31 18:00:54.816","last_synced_ts":"2024-05-31 18:00:47.766","now_ts":"2024-05-31 18:03:04.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ jq .synced ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:00:54.816","puller_resolved_ts":"2024-05-31' '18:00:54.816","last_synced_ts":"2024-05-31' '18:00:47.766","now_ts":"2024-05-31' '18:03:04.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + status=false + '[' false '!=' false ']' ++ jq -r .info ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:00:54.816","puller_resolved_ts":"2024-05-31' '18:00:54.816","last_synced_ts":"2024-05-31' '18:00:47.766","now_ts":"2024-05-31' '18:03:04.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386aafa00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:15986, start at 2024-05-31 18:03:25.590198879 +0800 CST m=+1.897086048 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:05:25.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:03:25.593 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:53:25.593 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386aafa00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:15986, start at 2024-05-31 18:03:25.590198879 +0800 CST m=+1.897086048 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:05:25.597 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:03:25.593 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:53:25.593 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386ab0580004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:16065, start at 2024-05-31 18:03:25.594585904 +0800 CST m=+1.835753893 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:05:25.601 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:03:25.590 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:53:25.590 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } start tidb cluster in /tmp/tidb_cdc_test/multi_topics Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.17423.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... + set +x + tso='450140519358529537 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140519358529537 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140519358529537 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri May 31 18:03:29 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1745617458.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:03:32 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66454501-6fe1-4d1a-b6d0-f1cf2f74738b {"id":"66454501-6fe1-4d1a-b6d0-f1cf2f74738b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149810} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1a8cd687 66454501-6fe1-4d1a-b6d0-f1cf2f74738b /tidb/cdc/default/default/upstream/7375102234742621052 {"id":7375102234742621052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66454501-6fe1-4d1a-b6d0-f1cf2f74738b {"id":"66454501-6fe1-4d1a-b6d0-f1cf2f74738b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149810} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1a8cd687 66454501-6fe1-4d1a-b6d0-f1cf2f74738b /tidb/cdc/default/default/upstream/7375102234742621052 {"id":7375102234742621052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/66454501-6fe1-4d1a-b6d0-f1cf2f74738b {"id":"66454501-6fe1-4d1a-b6d0-f1cf2f74738b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149810} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1a8cd687 66454501-6fe1-4d1a-b6d0-f1cf2f74738b /tidb/cdc/default/default/upstream/7375102234742621052 {"id":7375102234742621052,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140519358529537 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.17511.out cli changefeed create --start-ts=450140519358529537 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375102234742621052,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T18:03:33.293997438+08:00","start_ts":450140519358529537,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140519358529537,"checkpoint_ts":450140519358529537,"checkpoint_time":"2024-05-31 18:03:28.344"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386b3ed00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:31539, start at 2024-05-31 18:03:34.744918094 +0800 CST m=+1.857854313 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:05:34.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:03:34.758 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:53:34.758 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386b3ed00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:31539, start at 2024-05-31 18:03:34.744918094 +0800 CST m=+1.857854313 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:05:34.752 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:03:34.758 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:53:34.758 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386b403c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:31610, start at 2024-05-31 18:03:34.801213625 +0800 CST m=+1.854930597 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:05:34.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:03:34.799 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:53:34.799 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_topics/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_topics/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386b4b700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:19736, start at 2024-05-31 18:03:35.55205016 +0800 CST m=+1.743277290 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:05:35.559 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:03:35.516 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:53:35.516 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386b4b700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:19736, start at 2024-05-31 18:03:35.55205016 +0800 CST m=+1.743277290 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:05:35.559 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:03:35.516 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:53:35.516 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f386b4cd8000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:19802, start at 2024-05-31 18:03:35.619163439 +0800 CST m=+1.755807992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:05:35.625 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:03:35.606 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:53:35.606 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.cli.32912.out cli tso query --pd=http://127.0.0.1:2379 table test.t1 exists + sleep 5 + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21228.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140521760817153 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140521760817153 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 18:03:38 CST 2024] <<<<<< START cdc server in multi_topics case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.3294332945.out server --log-file /tmp/tidb_cdc_test/multi_topics/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='450140521998843905 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140521998843905 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140521998843905 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 18:03:39 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2126821270.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:03:42 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87decd0f-a228-40c6-932d-a7712a0f7dfd {"id":"87decd0f-a228-40c6-932d-a7712a0f7dfd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149819} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1ab4888a 87decd0f-a228-40c6-932d-a7712a0f7dfd /tidb/cdc/default/default/upstream/7375102270915420617 {"id":7375102270915420617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87decd0f-a228-40c6-932d-a7712a0f7dfd {"id":"87decd0f-a228-40c6-932d-a7712a0f7dfd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149819} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1ab4888a 87decd0f-a228-40c6-932d-a7712a0f7dfd /tidb/cdc/default/default/upstream/7375102270915420617 {"id":7375102270915420617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/87decd0f-a228-40c6-932d-a7712a0f7dfd {"id":"87decd0f-a228-40c6-932d-a7712a0f7dfd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149819} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1ab4888a 87decd0f-a228-40c6-932d-a7712a0f7dfd /tidb/cdc/default/default/upstream/7375102270915420617 {"id":7375102270915420617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics.cli.32985.out cli changefeed create --start-ts=450140521760817153 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics/conf/changefeed.toml Create changefeed successfully! ID: ff4ead71-4359-4ff4-b7d0-1f57d3d5a3b2 Info: {"upstream_id":7375102270915420617,"namespace":"default","id":"ff4ead71-4359-4ff4-b7d0-1f57d3d5a3b2","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-05-31T18:03:42.500624548+08:00","start_ts":450140521760817153,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"dispatchers":[{"matcher":["workload.*"],"topic":"workload"},{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140521760817153,"checkpoint_ts":450140521760817153,"checkpoint_time":"2024-05-31 18:03:37.508"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 15982 5.2 0.0 2726324 212828 ? Sl 18:03 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1717149803682615960.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb.log jenkins 15986 16.2 0.0 2644780 202680 ? Sl 18:03 0:02 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1717149803686116600.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_other.log jenkins 16065 16.5 0.0 2506620 230184 ? Sl 18:03 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1717149803751967407.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 6541 0 --:--:-- --:--:-- --:--:-- 6567 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 18:03:40.744","puller_resolved_ts":"2024-05-31 18:03:34.744","last_synced_ts":"2024-05-31 18:03:35.244","now_ts":"2024-05-31 18:03:42.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:03:40.744","puller_resolved_ts":"2024-05-31' '18:03:34.744","last_synced_ts":"2024-05-31' '18:03:35.244","now_ts":"2024-05-31' '18:03:42.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:03:40.744","puller_resolved_ts":"2024-05-31' '18:03:34.744","last_synced_ts":"2024-05-31' '18:03:35.244","now_ts":"2024-05-31' '18:03:42.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:03:42 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a455e6d-eda3-4af1-9e19-0bbebf931aad {"id":"3a455e6d-eda3-4af1-9e19-0bbebf931aad","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1ab46088 3a455e6d-eda3-4af1-9e19-0bbebf931aad /tidb/cdc/default/default/upstream/7375102278567056170 {"id":7375102278567056170,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a455e6d-eda3-4af1-9e19-0bbebf931aad {"id":"3a455e6d-eda3-4af1-9e19-0bbebf931aad","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1ab46088 3a455e6d-eda3-4af1-9e19-0bbebf931aad /tidb/cdc/default/default/upstream/7375102278567056170 {"id":7375102278567056170,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a455e6d-eda3-4af1-9e19-0bbebf931aad {"id":"3a455e6d-eda3-4af1-9e19-0bbebf931aad","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149820} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1ab46088 3a455e6d-eda3-4af1-9e19-0bbebf931aad /tidb/cdc/default/default/upstream/7375102278567056170 {"id":7375102278567056170,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140521998843905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21323.out cli changefeed create --start-ts=450140521998843905 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375102278567056170,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T18:03:43.3706043+08:00","start_ts":450140521998843905,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140521998843905,"checkpoint_ts":450140521998843905,"checkpoint_time":"2024-05-31 18:03:38.416"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 19091 38.9 0.5 4734692 2288784 ? Sl 18:03 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 19092 29.2 0.5 4692708 2228992 ? Sl 18:03 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 19093 29.2 0.5 4692712 2216584 ? Sl 18:03 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 19095 38.9 0.5 4723428 2273468 ? Sl 18:03 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 11113 0 --:--:-- --:--:-- --:--:-- 11571 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 18:03:50.866","puller_resolved_ts":"2024-05-31 18:03:44.816","last_synced_ts":"2024-05-31 18:03:44.816","now_ts":"2024-05-31 18:03:51.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:03:50.866","puller_resolved_ts":"2024-05-31' '18:03:44.816","last_synced_ts":"2024-05-31' '18:03:44.816","now_ts":"2024-05-31' '18:03:51.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:03:50.866","puller_resolved_ts":"2024-05-31' '18:03:44.816","last_synced_ts":"2024-05-31' '18:03:44.816","now_ts":"2024-05-31' '18:03:51.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + '[' kafka == mysql ']' + stop_tidb_cluster + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } 6:04PM INF > Run case=sql/debezium/skip_messages_test.sql 6:04PM INF > Run case=sql/debezium/strategy_test.sql [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } 6:04PM INF > Run case=sql/debezium/table_column_comment_test.sql 6:04PM INF > Run case=sql/debezium/timestamp_column_test.sql 6:04PM INF > Run case=sql/debezium/tinyint_test.sql 6:04PM INF > Run case=sql/debezium/topic_name_sanitization_test.sql 6:04PM INF > Run case=sql/debezium/unsigned_integer_test.sql [Fri May 31 18:05:44 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 1 [Fri May 31 18:05:44 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 2 [Fri May 31 18:05:44 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 3 [Fri May 31 18:05:44 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> table test.table1 not exists for 1-th check, retry later table test.table1 not exists for 2-th check, retry later table test.table1 exists table test.table2 exists table test.table3 exists check diff successfully table test.table10 not exists for 1-th check, retry later table test.table10 exists table test.table20 exists check diff successfully 6:05PM INF > Run case=sql/dml.sql ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 6968 0 --:--:-- --:--:-- --:--:-- 7129 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 18:05:50.844","puller_resolved_ts":"2024-05-31 18:05:44.844","last_synced_ts":"2024-05-31 18:03:35.244","now_ts":"2024-05-31 18:05:52.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '18:05:50.844","puller_resolved_ts":"2024-05-31' '18:05:44.844","last_synced_ts":"2024-05-31' '18:03:35.244","now_ts":"2024-05-31' '18:05:52.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '18:05:50.844","puller_resolved_ts":"2024-05-31' '18:05:44.844","last_synced_ts":"2024-05-31' '18:03:35.244","now_ts":"2024-05-31' '18:05:52.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 22597 0 --:--:-- --:--:-- --:--:-- 23322 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 18:03:51.866","puller_resolved_ts":"2024-05-31 18:03:51.866","last_synced_ts":"2024-05-31 18:03:44.816","now_ts":"2024-05-31 18:06:01.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:03:51.866","puller_resolved_ts":"2024-05-31' '18:03:51.866","last_synced_ts":"2024-05-31' '18:03:44.816","now_ts":"2024-05-31' '18:06:01.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:03:51.866","puller_resolved_ts":"2024-05-31' '18:03:51.866","last_synced_ts":"2024-05-31' '18:03:44.816","now_ts":"2024-05-31' '18:06:01.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3874e65c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:18803, start at 2024-05-31 18:06:12.919284263 +0800 CST m=+1.787787800 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:08:12.926 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:06:12.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:56:12.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3874e65c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:18803, start at 2024-05-31 18:06:12.919284263 +0800 CST m=+1.787787800 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:08:12.926 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:06:12.887 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:56:12.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3874e8940017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-bxzm1-rd03w, pid:18869, start at 2024-05-31 18:06:13.078106839 +0800 CST m=+1.892219314 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:08:13.086 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:06:13.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:56:13.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + cd /tmp/tidb_cdc_test/synced_status_with_redo + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.20181.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... + set +x + tso='450140563239600129 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140563239600129 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140563239600129 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Fri May 31 18:06:17 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.2020620208.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:06:20 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41fbead1-add5-45a7-8d5e-a2ffe9bed341 {"id":"41fbead1-add5-45a7-8d5e-a2ffe9bed341","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149977} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1d1ae882 41fbead1-add5-45a7-8d5e-a2ffe9bed341 /tidb/cdc/default/default/upstream/7375102957652363163 {"id":7375102957652363163,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41fbead1-add5-45a7-8d5e-a2ffe9bed341 {"id":"41fbead1-add5-45a7-8d5e-a2ffe9bed341","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149977} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1d1ae882 41fbead1-add5-45a7-8d5e-a2ffe9bed341 /tidb/cdc/default/default/upstream/7375102957652363163 {"id":7375102957652363163,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41fbead1-add5-45a7-8d5e-a2ffe9bed341 {"id":"41fbead1-add5-45a7-8d5e-a2ffe9bed341","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149977} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1d1ae882 41fbead1-add5-45a7-8d5e-a2ffe9bed341 /tidb/cdc/default/default/upstream/7375102957652363163 {"id":7375102957652363163,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140563239600129 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.20263.out cli changefeed create --start-ts=450140563239600129 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375102957652363163,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T18:06:20.680736394+08:00","start_ts":450140563239600129,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140563239600129,"checkpoint_ts":450140563239600129,"checkpoint_time":"2024-05-31 18:06:15.737"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + sleep 20 [Fri May 31 18:06:20 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 10 [Fri May 31 18:06:20 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: 20 [Fri May 31 18:06:20 CST 2024] <<<<<< START kafka consumer in multi_topics case >>>>>> schema registry uri found: finish table test.finish not exists for 1-th check, retry later table test.finish not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3875807c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:22510, start at 2024-05-31 18:06:22.784640931 +0800 CST m=+1.946056879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:08:22.792 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:06:22.751 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:56:22.751 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3875807c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:22510, start at 2024-05-31 18:06:22.784640931 +0800 CST m=+1.946056879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:08:22.792 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:06:22.751 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:56:22.751 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387581400006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:22578, start at 2024-05-31 18:06:22.807701641 +0800 CST m=+1.917427534 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:08:22.816 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:06:22.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:56:22.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.24001.out cli tso query --pd=http://127.0.0.1:2379 table test.finish exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:06:26 CST 2024] <<<<<< run test case multi_topics success! >>>>>> + set +x + tso='450140565799174145 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140565799174145 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140565799174145 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 18:06:26 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2403224034.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:06:30 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57ce9a18-2f6e-4b6b-ae7c-5bf2aa3c65f1 {"id":"57ce9a18-2f6e-4b6b-ae7c-5bf2aa3c65f1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149987} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1d40a78d 57ce9a18-2f6e-4b6b-ae7c-5bf2aa3c65f1 /tidb/cdc/default/default/upstream/7375102993184289978 {"id":7375102993184289978,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57ce9a18-2f6e-4b6b-ae7c-5bf2aa3c65f1 {"id":"57ce9a18-2f6e-4b6b-ae7c-5bf2aa3c65f1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149987} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1d40a78d 57ce9a18-2f6e-4b6b-ae7c-5bf2aa3c65f1 /tidb/cdc/default/default/upstream/7375102993184289978 {"id":7375102993184289978,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57ce9a18-2f6e-4b6b-ae7c-5bf2aa3c65f1 {"id":"57ce9a18-2f6e-4b6b-ae7c-5bf2aa3c65f1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717149987} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1d40a78d 57ce9a18-2f6e-4b6b-ae7c-5bf2aa3c65f1 /tidb/cdc/default/default/upstream/7375102993184289978 {"id":7375102993184289978,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140565799174145 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.24088.out cli changefeed create --start-ts=450140565799174145 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375102993184289978,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T18:06:30.462943445+08:00","start_ts":450140565799174145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140565799174145,"checkpoint_ts":450140565799174145,"checkpoint_time":"2024-05-31 18:06:25.501"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: kafka... <<================= Starting schema registry... * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 22506 4.8 0.0 2661112 210704 ? Sl 18:06 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717149980829728155.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 22510 15.9 0.0 2498492 238428 ? Sl 18:06 0:03 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717149980832670413.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 22578 16.4 0.0 2507312 249036 ? Sl 18:06 0:03 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1717149980883629998.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 4479 0 --:--:-- --:--:-- --:--:-- 4500 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 18:06:38.301","puller_resolved_ts":"2024-05-31 18:06:31.951","last_synced_ts":"2024-05-31 18:06:32.451","now_ts":"2024-05-31 18:06:39.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:06:38.301","puller_resolved_ts":"2024-05-31' '18:06:31.951","last_synced_ts":"2024-05-31' '18:06:32.451","now_ts":"2024-05-31' '18:06:39.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:06:38.301","puller_resolved_ts":"2024-05-31' '18:06:31.951","last_synced_ts":"2024-05-31' '18:06:32.451","now_ts":"2024-05-31' '18:06:39.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8088; Connection refused * Closing connection 0 * About to connect() to 127.0.0.1 port 8088 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8088 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8088 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:06:42 GMT < Content-Type: application/vnd.schemaregistry.v1+json < Vary: Accept-Encoding, User-Agent < Content-Length: 2 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact The 1 times to try to start tidb cluster... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 23467 0 --:--:-- --:--:-- --:--:-- 24100 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 18:06:15.737","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-31 18:06:42.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:06:15.737","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '18:06:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:06:15.737","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '18:06:42.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster start tidb cluster in /tmp/tidb_cdc_test/avro_basic Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 6:06PM INF > All tests pass failed=0 passed=219 Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + check_logs /tmp/tidb_cdc_test/synced_status_with_redo ++ date + echo '[Fri May 31 18:06:53 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>>' [Fri May 31 18:06:53 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>> + stop_tidb_cluster VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387767e80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:34632, start at 2024-05-31 18:06:53.952358384 +0800 CST m=+1.806478490 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:08:53.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:06:53.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:56:53.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387767e80006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:34632, start at 2024-05-31 18:06:53.952358384 +0800 CST m=+1.806478490 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:08:53.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:06:53.946 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:56:53.946 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387768ac0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:34706, start at 2024-05-31 18:06:54.013280335 +0800 CST m=+1.813754231 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:08:54.020 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:06:53.995 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:56:53.995 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/avro_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/avro_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/avro_basic/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 49 100 24 100 25 277 288 --:--:-- --:--:-- --:--:-- 290 {"compatibility":"NONE"}+ pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.cli.36130.out cli tso query --pd=http://127.0.0.1:2379 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Fri May 31 18:06:57 CST 2024] <<<<<< run test case debezium success! >>>>>> [Pipeline] // withCredentials [Pipeline] } + set +x + tso='450140574016077826 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140574016077826 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 18:06:58 CST 2024] <<<<<< START cdc server in avro_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.3616136163.out server --log-file /tmp/tidb_cdc_test/avro_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/avro_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:07:01 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a6ca1c3-a07d-4785-9fa2-df4b351865b3 {"id":"9a6ca1c3-a07d-4785-9fa2-df4b351865b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150018} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1dbb198a 9a6ca1c3-a07d-4785-9fa2-df4b351865b3 /tidb/cdc/default/default/upstream/7375103132254460267 {"id":7375103132254460267,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a6ca1c3-a07d-4785-9fa2-df4b351865b3 {"id":"9a6ca1c3-a07d-4785-9fa2-df4b351865b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150018} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1dbb198a 9a6ca1c3-a07d-4785-9fa2-df4b351865b3 /tidb/cdc/default/default/upstream/7375103132254460267 {"id":7375103132254460267,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a6ca1c3-a07d-4785-9fa2-df4b351865b3 {"id":"9a6ca1c3-a07d-4785-9fa2-df4b351865b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150018} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1dbb198a 9a6ca1c3-a07d-4785-9fa2-df4b351865b3 /tidb/cdc/default/default/upstream/7375103132254460267 {"id":7375103132254460267,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.avro_basic.cli.36216.out cli changefeed create --start-ts=450140574016077826 '--sink-uri=kafka://127.0.0.1:9092/ticdc-avro-test?protocol=avro&enable-tidb-extension=true&avro-enable-watermark=true&avro-decimal-handling-mode=string&avro-bigint-unsigned-handling-mode=string' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/avro_basic/conf/changefeed.toml --schema-registry=http://127.0.0.1:8088 Create changefeed successfully! ID: 8e9b865c-908b-49b0-a24a-cc749c762c41 Info: {"upstream_id":7375103132254460267,"namespace":"default","id":"8e9b865c-908b-49b0-a24a-cc749c762c41","sink_uri":"kafka://127.0.0.1:9092/ticdc-avro-test?protocol=avro\u0026enable-tidb-extension=true\u0026avro-enable-watermark=true\u0026avro-decimal-handling-mode=string\u0026avro-bigint-unsigned-handling-mode=string","create_time":"2024-05-31T18:07:01.855804273+08:00","start_ts":450140574016077826,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"avro","schema_registry":"http://127.0.0.1:8088","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"correctness","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140574016077826,"checkpoint_ts":450140574016077826,"checkpoint_time":"2024-05-31 18:06:56.846"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x [Fri May 31 18:07:03 CST 2024] <<<<<< START kafka consumer in avro_basic case >>>>>> schema registry uri found: http://127.0.0.1:8088 table test.finish_mark not exists for 1-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:07:11 CST 2024] <<<<<< run test case avro_basic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: kafka... <<================= [Fri May 31 18:07:11 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: kafka... <<================= [Fri May 31 18:07:14 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/canal_json_handle_key_only Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3879d7e40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:37431, start at 2024-05-31 18:07:33.903382596 +0800 CST m=+1.871972098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:09:33.911 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:07:33.881 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:57:33.881 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3879d7e40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:37431, start at 2024-05-31 18:07:33.903382596 +0800 CST m=+1.871972098 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:09:33.911 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:07:33.881 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:57:33.881 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3879d7180014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:37498, start at 2024-05-31 18:07:33.876221171 +0800 CST m=+1.793010510 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:09:33.883 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:07:33.881 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:57:33.881 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_handle_key_only/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.cli.38877.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140584458584065 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140584458584065 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 18:07:38 CST 2024] <<<<<< START cdc server in canal_json_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.3890238904.out server --log-file /tmp/tidb_cdc_test/canal_json_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_handle_key_only/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:07:41 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/900e376b-6d52-4836-9af2-2f0fa1b72724 {"id":"900e376b-6d52-4836-9af2-2f0fa1b72724","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1e58db88 900e376b-6d52-4836-9af2-2f0fa1b72724 /tidb/cdc/default/default/upstream/7375103300257458636 {"id":7375103300257458636,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/900e376b-6d52-4836-9af2-2f0fa1b72724 {"id":"900e376b-6d52-4836-9af2-2f0fa1b72724","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1e58db88 900e376b-6d52-4836-9af2-2f0fa1b72724 /tidb/cdc/default/default/upstream/7375103300257458636 {"id":7375103300257458636,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/900e376b-6d52-4836-9af2-2f0fa1b72724 {"id":"900e376b-6d52-4836-9af2-2f0fa1b72724","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150058} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1e58db88 900e376b-6d52-4836-9af2-2f0fa1b72724 /tidb/cdc/default/default/upstream/7375103300257458636 {"id":7375103300257458636,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_handle_key_only.cli.38954.out cli changefeed create --start-ts=450140584458584065 '--sink-uri=kafka://127.0.0.1:9092/canal-json-handle-key-only?protocol=canal-json&enable-tidb-extension=true&max-message-bytes=1000&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: cdc41c70-6126-4f6a-9cc5-e7e4caefdf05 Info: {"upstream_id":7375103300257458636,"namespace":"default","id":"cdc41c70-6126-4f6a-9cc5-e7e4caefdf05","sink_uri":"kafka://127.0.0.1:9092/canal-json-handle-key-only?protocol=canal-json\u0026enable-tidb-extension=true\u0026max-message-bytes=1000\u0026kafka-version=2.4.1","create_time":"2024-05-31T18:07:41.63807693+08:00","start_ts":450140584458584065,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"snappy","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140584458584065,"checkpoint_ts":450140584458584065,"checkpoint_time":"2024-05-31 18:07:36.681"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:07:50 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh: line 1: 38992 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/open_protocol_handle_key_only Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387c47640010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:40150, start at 2024-05-31 18:08:13.803051616 +0800 CST m=+1.870571989 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:10:13.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:08:13.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:58:13.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387c47640010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:40150, start at 2024-05-31 18:08:13.803051616 +0800 CST m=+1.870571989 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:10:13.810 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:08:13.785 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:58:13.785 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387c48080015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:40225, start at 2024-05-31 18:08:13.870808046 +0800 CST m=+1.871025133 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:10:13.878 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:08:13.875 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:58:13.875 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/open_protocol_handle_key_only/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.cli.41584.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140594919178241 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140594919178241 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 18:08:18 CST 2024] <<<<<< START cdc server in open_protocol_handle_key_only case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.4162741629.out server --log-file /tmp/tidb_cdc_test/open_protocol_handle_key_only/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/open_protocol_handle_key_only/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:08:21 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57e88d49-f3d1-4606-ab9c-75d4c6cd8fcf {"id":"57e88d49-f3d1-4606-ab9c-75d4c6cd8fcf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150098} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1ef4b986 57e88d49-f3d1-4606-ab9c-75d4c6cd8fcf /tidb/cdc/default/default/upstream/7375103474266700597 {"id":7375103474266700597,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57e88d49-f3d1-4606-ab9c-75d4c6cd8fcf {"id":"57e88d49-f3d1-4606-ab9c-75d4c6cd8fcf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150098} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1ef4b986 57e88d49-f3d1-4606-ab9c-75d4c6cd8fcf /tidb/cdc/default/default/upstream/7375103474266700597 {"id":7375103474266700597,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/57e88d49-f3d1-4606-ab9c-75d4c6cd8fcf {"id":"57e88d49-f3d1-4606-ab9c-75d4c6cd8fcf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150098} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1ef4b986 57e88d49-f3d1-4606-ab9c-75d4c6cd8fcf /tidb/cdc/default/default/upstream/7375103474266700597 {"id":7375103474266700597,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_handle_key_only.cli.41668.out cli changefeed create --start-ts=450140594919178241 '--sink-uri=kafka://127.0.0.1:9092/open-protocol-handle-key-only?protocol=open-protocol&max-message-bytes=800&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/conf/changefeed.toml Create changefeed successfully! ID: 39b1244b-ec59-48f8-9d13-3682a45edcf6 Info: {"upstream_id":7375103474266700597,"namespace":"default","id":"39b1244b-ec59-48f8-9d13-3682a45edcf6","sink_uri":"kafka://127.0.0.1:9092/open-protocol-handle-key-only?protocol=open-protocol\u0026max-message-bytes=800\u0026kafka-version=2.4.1","create_time":"2024-05-31T18:08:21.594327105+08:00","start_ts":450140594919178241,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"handle-key-only","large_message_handle_compression":"lz4","claim_check_storage_uri":""}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140594919178241,"checkpoint_ts":450140594919178241,"checkpoint_time":"2024-05-31 18:08:16.585"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later table test.finish_mark not exists for 3-th check, retry later table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:08:30 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh: line 1: 41710 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/canal_json_claim_check Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387ea9a40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:42959, start at 2024-05-31 18:08:52.858909693 +0800 CST m=+1.866464333 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:10:52.866 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:08:52.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:58:52.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387ea9a40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:42959, start at 2024-05-31 18:08:52.858909693 +0800 CST m=+1.866464333 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:10:52.866 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:08:52.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:58:52.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387ea99c000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:43033, start at 2024-05-31 18:08:52.851562868 +0800 CST m=+1.803103946 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:10:52.859 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:08:52.839 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:58:52.839 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_claim_check/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.cli.44363.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140605682024449 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140605682024449 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 18:08:59 CST 2024] <<<<<< START cdc server in canal_json_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.4439544397.out server --log-file /tmp/tidb_cdc_test/canal_json_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_claim_check/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 9400 0 --:--:-- --:--:-- --:--:-- 9608 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-31 18:08:48.452","puller_resolved_ts":"2024-05-31 18:08:41.451","last_synced_ts":"2024-05-31 18:06:32.451","now_ts":"2024-05-31 18:08:49.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '18:08:48.452","puller_resolved_ts":"2024-05-31' '18:08:41.451","last_synced_ts":"2024-05-31' '18:06:32.451","now_ts":"2024-05-31' '18:08:49.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-31' '18:08:48.452","puller_resolved_ts":"2024-05-31' '18:08:41.451","last_synced_ts":"2024-05-31' '18:06:32.451","now_ts":"2024-05-31' '18:08:49.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:09:02 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae9187ae-fabe-493e-abea-2a83d8d4906b {"id":"ae9187ae-fabe-493e-abea-2a83d8d4906b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150139} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1f913290 ae9187ae-fabe-493e-abea-2a83d8d4906b /tidb/cdc/default/default/upstream/7375103640040506184 {"id":7375103640040506184,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae9187ae-fabe-493e-abea-2a83d8d4906b {"id":"ae9187ae-fabe-493e-abea-2a83d8d4906b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150139} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1f913290 ae9187ae-fabe-493e-abea-2a83d8d4906b /tidb/cdc/default/default/upstream/7375103640040506184 {"id":7375103640040506184,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae9187ae-fabe-493e-abea-2a83d8d4906b {"id":"ae9187ae-fabe-493e-abea-2a83d8d4906b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150139} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1f913290 ae9187ae-fabe-493e-abea-2a83d8d4906b /tidb/cdc/default/default/upstream/7375103640040506184 {"id":7375103640040506184,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_claim_check.cli.44443.out cli changefeed create --start-ts=450140605682024449 '--sink-uri=kafka://127.0.0.1:9092/canal-json-claim-check?protocol=canal-json&enable-tidb-extension=true&max-message-bytes=1000&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/conf/changefeed.toml chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Create changefeed successfully! ID: a96a5d8c-e94e-4a50-900a-6e2e563652a5 Info: {"upstream_id":7375103640040506184,"namespace":"default","id":"a96a5d8c-e94e-4a50-900a-6e2e563652a5","sink_uri":"kafka://127.0.0.1:9092/canal-json-claim-check?protocol=canal-json\u0026enable-tidb-extension=true\u0026max-message-bytes=1000\u0026kafka-version=2.4.1","create_time":"2024-05-31T18:09:02.636141135+08:00","start_ts":450140605682024449,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"snappy","claim_check_storage_uri":"file:///tmp/canal-json-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140605682024449,"checkpoint_ts":450140605682024449,"checkpoint_time":"2024-05-31 18:08:57.642"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... + set +x table test.finish_mark not exists for 1-th check, retry later Verifying downstream PD is started... table test.finish_mark not exists for 2-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:09:09 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387fc9400017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:25397, start at 2024-05-31 18:09:11.266485151 +0800 CST m=+2.410654555 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:11:11.275 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:09:11.248 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:59:11.248 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387fc9400017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:25397, start at 2024-05-31 18:09:11.266485151 +0800 CST m=+2.410654555 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:11:11.275 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:09:11.248 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:59:11.248 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f387fc8a40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-lxbz5-3b3rn, pid:25474, start at 2024-05-31 18:09:11.251973546 +0800 CST m=+2.334193029 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:11:11.261 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:09:11.260 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:59:11.260 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26784.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140610375974913 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140610375974913 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=450140610375974913 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Fri May 31 18:09:17 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2681826820.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:09:20 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1be8f012-b7f4-40d2-8fea-ef2fd6665a65 {"id":"1be8f012-b7f4-40d2-8fea-ef2fd6665a65","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150157} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1fd0b29a 1be8f012-b7f4-40d2-8fea-ef2fd6665a65 /tidb/cdc/default/default/upstream/7375103711264340106 {"id":7375103711264340106,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1be8f012-b7f4-40d2-8fea-ef2fd6665a65 {"id":"1be8f012-b7f4-40d2-8fea-ef2fd6665a65","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150157} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1fd0b29a 1be8f012-b7f4-40d2-8fea-ef2fd6665a65 /tidb/cdc/default/default/upstream/7375103711264340106 {"id":7375103711264340106,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1be8f012-b7f4-40d2-8fea-ef2fd6665a65 {"id":"1be8f012-b7f4-40d2-8fea-ef2fd6665a65","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150157} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce1fd0b29a 1be8f012-b7f4-40d2-8fea-ef2fd6665a65 /tidb/cdc/default/default/upstream/7375103711264340106 {"id":7375103711264340106,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=450140610375974913 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.26862.out cli changefeed create --start-ts=450140610375974913 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7375103711264340106,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-31T18:09:20.523888246+08:00","start_ts":450140610375974913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140610375974913,"checkpoint_ts":450140610375974913,"checkpoint_time":"2024-05-31 18:09:15.548"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh: line 1: 44484 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: kafka... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/open_protocol_claim_check Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3881230c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:45681, start at 2024-05-31 18:09:33.418213018 +0800 CST m=+1.892308177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:11:33.424 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:09:33.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:59:33.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f3881230c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:45681, start at 2024-05-31 18:09:33.418213018 +0800 CST m=+1.892308177 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:11:33.424 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:09:33.429 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:59:33.429 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f388122f8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:45748, start at 2024-05-31 18:09:33.392118114 +0800 CST m=+1.806683941 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:11:33.399 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:09:33.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-17:59:33.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/open_protocol_claim_check/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.47053.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='450140615784267777 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140615784267777 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.47107.out cli tso query --pd=http://127.0.0.1:2379 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 22938 0 --:--:-- --:--:-- --:--:-- 23322 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-31 18:09:15.548","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-31 18:09:41.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:09:15.548","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '18:09:41.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-31' '18:09:15.548","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-31' '18:09:41.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + set +x + tso='450140617566846977 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...' + echo 450140617566846977 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Fri May 31 18:09:44 CST 2024] <<<<<< START cdc server in open_protocol_claim_check case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.4714247144.out server --log-file /tmp/tidb_cdc_test/open_protocol_claim_check/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/open_protocol_claim_check/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:09:47 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/000c2c88-cdfe-4b35-b014-3605fd62939d {"id":"000c2c88-cdfe-4b35-b014-3605fd62939d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150184} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce2029aff2 000c2c88-cdfe-4b35-b014-3605fd62939d /tidb/cdc/default/default/upstream/7375103816798746614 {"id":7375103816798746614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/000c2c88-cdfe-4b35-b014-3605fd62939d {"id":"000c2c88-cdfe-4b35-b014-3605fd62939d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150184} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce2029aff2 000c2c88-cdfe-4b35-b014-3605fd62939d /tidb/cdc/default/default/upstream/7375103816798746614 {"id":7375103816798746614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/000c2c88-cdfe-4b35-b014-3605fd62939d {"id":"000c2c88-cdfe-4b35-b014-3605fd62939d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150184} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce2029aff2 000c2c88-cdfe-4b35-b014-3605fd62939d /tidb/cdc/default/default/upstream/7375103816798746614 {"id":7375103816798746614,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.open_protocol_claim_check.cli.47190.out cli changefeed create --start-ts=450140615784267777 --target-ts=450140617566846977 '--sink-uri=kafka://127.0.0.1:9092/open-protocol-claim-check?protocol=open-protocol&max-message-bytes=800&kafka-version=2.4.1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/conf/changefeed.toml Create changefeed successfully! ID: 348f9b40-ded9-4b29-b088-4ef96abcb4a7 Info: {"upstream_id":7375103816798746614,"namespace":"default","id":"348f9b40-ded9-4b29-b088-4ef96abcb4a7","sink_uri":"kafka://127.0.0.1:9092/open-protocol-claim-check?protocol=open-protocol\u0026max-message-bytes=800\u0026kafka-version=2.4.1","create_time":"2024-05-31T18:09:47.966658654+08:00","start_ts":450140615784267777,"target_ts":450140617566846977,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"kafka_config":{"large_message_handle":{"large_message_handle_option":"claim-check","large_message_handle_compression":"lz4","claim_check_storage_uri":"file:///tmp/open-protocol-claim-check"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140615784267777,"checkpoint_ts":450140615784267777,"checkpoint_time":"2024-05-31 18:09:36.179"} PASS coverage: 2.6% of statements in github.com/pingcap/tiflow/... + set +x table test.finish_mark not exists for 1-th check, retry later table test.finish_mark not exists for 2-th check, retry later + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Fri May 31 18:09:53 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Fri May 31 18:09:53 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster table test.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:09:55 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh: line 1: 47227 Killed cdc_kafka_consumer --upstream-uri $SINK_URI --downstream-uri="mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false" --upstream-tidb-dsn="root@tcp(${UP_TIDB_HOST}:${UP_TIDB_PORT})/?" --config="$CUR/conf/changefeed.toml" 2>&1 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: kafka... <<================= [Fri May 31 18:10:06 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: kafka... <<================= [Fri May 31 18:10:10 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: kafka... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17D48B060F971C08 < X-Xss-Protection: 1; mode=block < Date: Fri, 31 May 2024 10:10:16 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Starting Downstream PD... Release Version: v8.2.0-alpha-64-g199b01792 Edition: Community Git Commit Hash: 199b01792159e5d8e83ef419a5053401e998bb0e Git Branch: master UTC Build Time: 2024-05-31 08:34:21 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 7ccbe3144fd22b35eece56a8125734a7bc2f7502 Git Commit Branch: master UTC Build Time: 2024-05-30 07:21:49 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-275-g35e09a2a36 Edition: Community Git Commit Hash: 35e09a2a36d227c6b45314fff0502ab93fab8aba Git Branch: master UTC Build Time: 2024-05-31 08:38:26 GoVersion: go1.21.10 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f388467640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:48615, start at 2024-05-31 18:10:26.934742347 +0800 CST m=+1.806242666 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:12:26.941 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:10:26.905 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-18:00:26.905 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f388467640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:48615, start at 2024-05-31 18:10:26.934742347 +0800 CST m=+1.806242666 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:12:26.941 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:10:26.905 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-18:00:26.905 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 198 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63f388468e40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-kafka-test-2156-9nv9v-j9lhd, pid:48684, start at 2024-05-31 18:10:27.038327093 +0800 CST m=+1.846125580 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240531-18:12:27.047 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240531-18:10:27.051 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240531-18:00:27.051 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-50-g640f856eb Edition: Community Git Commit Hash: 640f856eb25e246e81f6e3e4dec0c64c32a385f3 Git Branch: HEAD UTC Build Time: 2024-05-31 07:39:42 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-05-31 07:43:26 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-50-g640f856eb"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["640f856eb25e246e81f6e3e4dec0c64c32a385f3"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Fri May 31 18:10:29 CST 2024] <<<<<< START cdc server in multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl.5008150083.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Fri, 31 May 2024 10:10:32 GMT < Content-Length: 815 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6825e9cf-ecd7-41bd-ac6e-b2904798ae7a {"id":"6825e9cf-ecd7-41bd-ac6e-b2904798ae7a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150229} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce20fedf80 6825e9cf-ecd7-41bd-ac6e-b2904798ae7a /tidb/cdc/default/default/upstream/7375104049868507084 {"id":7375104049868507084,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6825e9cf-ecd7-41bd-ac6e-b2904798ae7a {"id":"6825e9cf-ecd7-41bd-ac6e-b2904798ae7a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150229} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce20fedf80 6825e9cf-ecd7-41bd-ac6e-b2904798ae7a /tidb/cdc/default/default/upstream/7375104049868507084 {"id":7375104049868507084,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6825e9cf-ecd7-41bd-ac6e-b2904798ae7a {"id":"6825e9cf-ecd7-41bd-ac6e-b2904798ae7a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-75-g18cf66054","git-hash":"18cf66054053b7e9a688825ac1a73321e0b6ee8d","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_kafka_test/tiflow/bin/cdc.test","start-timestamp":1717150229} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318fce20fedf80 6825e9cf-ecd7-41bd-ac6e-b2904798ae7a /tidb/cdc/default/default/upstream/7375104049868507084 {"id":7375104049868507084,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7375104049868507084,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-10587?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T18:10:32.82091548+08:00","start_ts":450140629763620865,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140629763620865,"checkpoint_ts":450140629763620865,"checkpoint_time":"2024-05-31 18:10:29.506"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7375104049868507084,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-8685?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T18:10:33.015742238+08:00","start_ts":450140629763620865,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140629763620865,"checkpoint_ts":450140629763620865,"checkpoint_time":"2024-05-31 18:10:29.506"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7375104049868507084,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-29111?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-31T18:10:33.216742851+08:00","start_ts":450140629763620865,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-75-g18cf66054","resolved_ts":450140629763620865,"checkpoint_ts":450140629763620865,"checkpoint_time":"2024-05-31 18:10:29.506"} [Fri May 31 18:10:33 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> [Fri May 31 18:10:33 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> [Fri May 31 18:10:33 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl case >>>>>> table multi_tables_ddl_test.t55 not exists for 1-th check, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later table multi_tables_ddl_test.t55 not exists for 5-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s + info='{ "upstream_id": 7375104049868507084, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450140631008542744, "checkpoint_time": "2024-05-31 18:10:34.255", "error": null }' + echo '{ "upstream_id": 7375104049868507084, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450140631008542744, "checkpoint_time": "2024-05-31 18:10:34.255", "error": null }' { "upstream_id": 7375104049868507084, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 450140631008542744, "checkpoint_time": "2024-05-31 18:10:34.255", "error": null } ++ echo '{' '"upstream_id":' 7375104049868507084, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450140631008542744, '"checkpoint_time":' '"2024-05-31' '18:10:34.255",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375104049868507084, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 450140631008542744, '"checkpoint_time":' '"2024-05-31' '18:10:34.255",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7375104049868507084, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450140634652344324, "checkpoint_time": "2024-05-31 18:10:48.155", "error": null }' + echo '{ "upstream_id": 7375104049868507084, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450140634652344324, "checkpoint_time": "2024-05-31 18:10:48.155", "error": null }' { "upstream_id": 7375104049868507084, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 450140634652344324, "checkpoint_time": "2024-05-31 18:10:48.155", "error": null } ++ echo '{' '"upstream_id":' 7375104049868507084, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450140634652344324, '"checkpoint_time":' '"2024-05-31' '18:10:48.155",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7375104049868507084, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 450140634652344324, '"checkpoint_time":' '"2024-05-31' '18:10:48.155",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s + info='{ "upstream_id": 7375104049868507084, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450140630851256366, "checkpoint_time": "2024-05-31 18:10:33.655", "error": { "time": "2024-05-31T18:10:35.180714825+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7375104049868507084, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450140630851256366, "checkpoint_time": "2024-05-31 18:10:33.655", "error": { "time": "2024-05-31T18:10:35.180714825+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7375104049868507084, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 450140630851256366, "checkpoint_time": "2024-05-31 18:10:33.655", "error": { "time": "2024-05-31T18:10:35.180714825+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7375104049868507084, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450140630851256366, '"checkpoint_time":' '"2024-05-31' '18:10:33.655",' '"error":' '{' '"time":' '"2024-05-31T18:10:35.180714825+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7375104049868507084, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 450140630851256366, '"checkpoint_time":' '"2024-05-31' '18:10:33.655",' '"error":' '{' '"time":' '"2024-05-31T18:10:35.180714825+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Fri May 31 18:10:50 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_kafka_test-2156/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS